{"meta":{"date":"2026-09-24","timezone":"UTC","generated_at":"2026-10-14T19:14:02Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":123,"failed_runs":94,"runs_with_occurrences":94,"runs_with_failure_pattern_matches":91,"runs_with_unmatched_occurrences":3,"runs_without_occurrence_rows":29,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7146/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103270905017798656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7146,"pr_state":"closed","pr_sha":"210214039c8925fdc446fab903f1f8d1a21e74ae","final_merged_sha":"34a27988a2da6ca8b624fbab48af075e636b7eb6","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T23:50:35Z","started_at":"2026-09-24T23:50:35Z","completed_at":"2026-09-25T00:06:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T00:18:03.923736732Z","timing_metadata_checked_at":"2026-09-25T00:18:03.923736732Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T00:18:03.926429723Z","region_metadata_checked_at":"2026-09-25T00:18:03.926429723Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"14f3f7f61957ab0fa541100fc89e340c6a8b2897f58ff9b85baa4d55ec5d284f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7146/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103270905017798656","occurred_at":"2026-09-24T23:50:35Z","failed_at":"provision","signature_id":"8b0f84d3fd13e52b773c2d5762e6e413e0d4ff59cb390c1fe435499d6e4b2b92","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/kusto/maestroMqttDataConnection","test_suite":"templatize-pipeline","failure_text":"time=2026-09-24T23:55:56.171Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection description=\"Step maestroMqttDataConnection\\n Kind: ARM\\n Template: templates/maestro-mqtt-data-connection.bicep\\n Parameters: configurations/maestro-mqtt-data-connection.tmpl.bicepparam\"\ntime=2026-09-24T23:55:56.789Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection\ntime=2026-09-24T23:55:57.168Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection err=\"failed to run ARM step: failed to create deployment: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-kusto-us/providers/Microsoft.Resources/deployments/23db75d3c4a6f5fe32de7df35567f808137cc8acac2d401192c396cc43f1121e\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidTemplate\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidTemplate\\\",\\n \\\"message\\\": \\\"Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'maestroMqttEventHubId'. The parameters defined in the template are: 'databaseName, eventHubId, eventhubEnabled, kustoConsumerGroupName, kustoDataConnectionName, kustoEnabled, kustoName, location'. Please see https://aka.ms/arm-pass-parameter-values for usage details.'.\\\",\\n \\\"additionalInfo\\\": [\\n {\\n \\\"type\\\": \\\"TemplateViolation\\\",\\n \\\"info\\\": {\\n \\\"lineNumber\\\": 0,\\n \\\"linePosition\\\": 0,\\n \\\"path\\\": \\\"\\\"\\n }\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0905a6f0720326490c59375e0404554c58e1e0996cc236e2a705a956863de47f","failure_pattern":"ERROR CODE: InvalidTemplate; detail message Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'maestroMqttEventHubId'. The parameters defined in the template are: 'databaseName, eventHubId, eventhubEnabled, kustoConsumerGroupName, kustoDataConnectionName, kustoEnabled, kustoName, location'.","search_query":"ERROR CODE: InvalidTemplate"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0905a6f0720326490c59375e0404554c58e1e0996cc236e2a705a956863de47f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103270655540596736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6846,"pr_state":"open","pr_sha":"99ee68d823b5d9607d6d356b0b4ee40d3ed8f7bd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T23:49:35Z","started_at":"2026-09-24T23:49:35Z","completed_at":"2026-09-25T02:02:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T02:13:49.605590021Z","timing_metadata_checked_at":"2026-09-25T02:13:49.605590021Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T02:13:49.608847411Z","region_metadata_checked_at":"2026-09-25T02:13:49.608847411Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"938bfa87567d5031ba1f88e0b58954ad42d5953066b0659dc780444456b4705e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103270655540596736","occurred_at":"2026-09-24T23:49:35Z","failed_at":"alert","signature_id":"cbcc41fb87ceb239baf4c483d3ab5268cdfd05b5c1837f310e5d12738e55e904","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T01:15:33Z\n Ended: 2026-09-25T01:24:33Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0596736-mgmt-2\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-646d8d7d48-fmq9z\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-646d8d7d48-fmq9z (namespace kube-applier) on cluster ci01-j0596736-mgmt-2 has averaged 129.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103269477926834176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6969,"pr_state":"closed","pr_sha":"47b70933d4d43a54ff5546771e596454861c2a27","final_merged_sha":"6dc92c0e1eae82818633321751269a6063e49fa2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T23:44:55Z","started_at":"2026-09-24T23:44:55Z","completed_at":"2026-09-25T01:55:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T02:03:28.971200167Z","timing_metadata_checked_at":"2026-09-25T02:03:28.971200167Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T02:03:28.975723852Z","region_metadata_checked_at":"2026-09-25T02:03:28.975723852Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"ac0c7ba010113a59071b02e3d12b238d2c5ab2e17c2b483f3125a3b7e06a6dce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103269477926834176","occurred_at":"2026-09-24T23:44:55Z","failed_at":"alert","signature_id":"6f2b6a144e3541517f2630051f3cefcfffdb5893b1112dd3f933dca064c7ef54","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T00:29:48Z\n Ended: 2026-09-25T01:02:49Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6834176-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6834176-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103269397211648000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7118,"pr_state":"closed","pr_sha":"179ae75a7e3eff02b934146a76b63b8d8d485193","final_merged_sha":"f23942d30dc47ec7edeebb24ec30f183a59ea2fd","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T23:44:35Z","started_at":"2026-09-24T23:44:35Z","completed_at":"2026-09-25T00:28:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T00:39:26.220893016Z","timing_metadata_checked_at":"2026-09-25T00:39:26.220893016Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T00:39:26.226583198Z","region_metadata_checked_at":"2026-09-25T00:39:26.226583198Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"f5a30b34160c8f3622f4a4e975ed0051a8fc56590ca16f198b318c15f2b93536","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103269397211648000","occurred_at":"2026-09-24T23:44:35Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-24T23:48:24.249Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T23:48:25.676Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T23:48:26.816Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9fe92f2075d3a162115703f2a2dd0a40f461263d369dfb42681221c1b82cb479 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1648000%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9fe92f2075d3a162115703f2a2dd0a40f461263d369dfb42681221c1b82cb479\ntime=2026-09-24T23:50:57.493Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1648000/providers/Microsoft.Resources/deployments/9fe92f2075d3a162115703f2a2dd0a40f461263d369dfb42681221c1b82cb479/operationStatuses/08584113131791545522\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1648000/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1648000/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1648000\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report a6a8d0ae-c018-4639-a518-3ed1c7beb811:9/24/2026 11:49:54 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T23:52:57.493Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T23:52:58.386Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T23:52:59.129Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=cf6e37d6cff770de12dbdbffa8c4623b3a737efd76d033bcf339dec1a4fb04e1 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1648000%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fcf6e37d6cff770de12dbdbffa8c4623b3a737efd76d033bcf339dec1a4fb04e1\ntime=2026-09-25T00:09:04.051Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1648000/providers/Microsoft.Resources/deployments/cf6e37d6cff770de12dbdbffa8c4623b3a737efd76d033bcf339dec1a4fb04e1/operationStatuses/08584113129068388577\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1648000/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1648000/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1648000\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report fc9f1334-cb66-4a1b-a69a-b44d3f0d15c9:9/25/2026 12:08:37 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T00:11:04.059Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T00:11:04.960Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T00:11:05.763Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=a4eb99266d655f9a85d7de315dc769d2d5008a2127b29773f350744d3fa412d7 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1648000%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fa4eb99266d655f9a85d7de315dc769d2d5008a2127b29773f350744d3fa412d7\ntime=2026-09-25T00:18:24.009Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7151/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103259408925462528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7151,"pr_state":"closed","pr_sha":"3bff2da523d9e5eee57af4c697c295999285cdfa","final_merged_sha":"3bff2da523d9e5eee57af4c697c295999285cdfa","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-24T23:04:54Z","started_at":"2026-09-24T23:04:54Z","completed_at":"2026-09-25T01:11:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T01:21:08.133989078Z","timing_metadata_checked_at":"2026-09-25T01:21:08.133989078Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T01:21:08.138576563Z","region_metadata_checked_at":"2026-09-25T01:21:08.138576563Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103259159142076416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7133,"pr_state":"open","pr_sha":"c2d1719d2c83ce4eb02f5e7803f798b5b01e417f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T23:03:54Z","started_at":"2026-09-24T23:03:54Z","completed_at":"2026-09-24T23:46:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T23:57:09.626004548Z","timing_metadata_checked_at":"2026-09-24T23:57:09.626004548Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T23:57:09.629362838Z","region_metadata_checked_at":"2026-09-24T23:57:09.629362838Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5c11ef145e5b6efb6381fd1a2159f247046f0babe13110b867cf1f374f70cf66","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103259159142076416","occurred_at":"2026-09-24T23:03:54Z","failed_at":"provision","signature_id":"36808247d658b9b22b59f418949b27fa87b1655933f39638ef6bec47bfa0327e","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-24T23:38:34.810Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 description=\"Step mitigations\\n Kind: Helm\\n\"\ntime=2026-09-24T23:38:35.282Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 values=\"map[azureCnsCpuLimits:map[enabled:true] jobImage:mcr.microsoft.com/aks/command/runtime@sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4]\"\ntime=2026-09-24T23:38:35.282Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-09-24T23:38:35.513Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 namespace=aro-hcp-mitigations\ntime=2026-09-24T23:38:35.703Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 dryRun=false\ntime=2026-09-24T23:38:35.703Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-09-24T23:38:35.755Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-09-24T23:39:19.239Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err=\"failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-09-24T23:39:19.239Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abb1429e70773e39316bd0948d4a9486cc702f42873918d630fe98006b70e184","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["abb1429e70773e39316bd0948d4a9486cc702f42873918d630fe98006b70e184"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103258851598929920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6968,"pr_state":"open","pr_sha":"1b77253f5e7b2f21b8c9906437e2f353554c3ddd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-24T23:02:41Z","started_at":"2026-09-24T23:02:41Z","completed_at":"2026-09-25T01:18:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T01:31:55.234655566Z","timing_metadata_checked_at":"2026-09-25T01:31:55.234655566Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T01:31:55.237552356Z","region_metadata_checked_at":"2026-09-25T01:31:55.237552356Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7150/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103255953611689984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7150,"pr_state":"closed","pr_sha":"7a563cc5f0275dd24c6edaf42691aeac6d6897a5","final_merged_sha":"5fdb2bd77b96d67d094d207e504542b274717b8f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T22:51:10Z","started_at":"2026-09-24T22:51:10Z","completed_at":"2026-09-24T23:42:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T23:57:10.053312499Z","timing_metadata_checked_at":"2026-09-24T23:57:10.053312499Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T23:57:10.056181291Z","region_metadata_checked_at":"2026-09-24T23:57:10.056181291Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"59a9d88abc297995742c1423e6874fd7c429313e9e477eb65fcdc9df68ab5136","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7150/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103255953611689984","occurred_at":"2026-09-24T22:51:10Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-24T23:03:28.056Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T23:03:29.473Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T23:03:30.301Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=12f2d669fa06dd4de2ba8409d1e9b7aa7f63bae2d0c78ba209c8d1082b4c957e portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1689984%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F12f2d669fa06dd4de2ba8409d1e9b7aa7f63bae2d0c78ba209c8d1082b4c957e\ntime=2026-09-24T23:06:00.848Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1689984/providers/Microsoft.Resources/deployments/12f2d669fa06dd4de2ba8409d1e9b7aa7f63bae2d0c78ba209c8d1082b4c957e/operationStatuses/08584113158756314414\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1689984/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1689984/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1689984\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 6bb9bafd-0fb4-4aa8-a6e2-b2e4391847ba:9/24/2026 11:04:31 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T23:08:00.850Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T23:08:01.497Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T23:08:02.303Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=4b6dbf996876bb5363df66ec3bbff19c39d2fbac30f092592bf67f3fff9b6f47 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1689984%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F4b6dbf996876bb5363df66ec3bbff19c39d2fbac30f092592bf67f3fff9b6f47\ntime=2026-09-24T23:24:36.647Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1689984/providers/Microsoft.Resources/deployments/4b6dbf996876bb5363df66ec3bbff19c39d2fbac30f092592bf67f3fff9b6f47/operationStatuses/08584113156036989890\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1689984/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1689984/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1689984\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 7e35e8ae-d5b1-4064-b001-fb471f9768fb:9/24/2026 11:23:59 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T23:26:36.652Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T23:26:37.173Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T23:26:37.799Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=1b5feed18dc2aa2598959e31b9190f5f385e888a1a154d00d07d2ff191153df4 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1689984%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F1b5feed18dc2aa2598959e31b9190f5f385e888a1a154d00d07d2ff191153df4\ntime=2026-09-24T23:33:27.754Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7145/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103255589084729344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7145,"pr_state":"closed","pr_sha":"11b7d3e6c3699b4bc8b6f2c3c485e0f0c744c91a","final_merged_sha":"11b7d3e6c3699b4bc8b6f2c3c485e0f0c744c91a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T22:49:43Z","started_at":"2026-09-24T22:49:43Z","completed_at":"2026-09-25T00:56:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T01:10:56.565853872Z","timing_metadata_checked_at":"2026-09-25T01:10:56.565853872Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T01:10:56.568427564Z","region_metadata_checked_at":"2026-09-25T01:10:56.568427564Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"3adb0b08c3a478c936cf85910aa77f0ea06cf55275a91239e2f5c85f560bfd5c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7145/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103255589084729344","occurred_at":"2026-09-24T22:49:43Z","failed_at":"alert","signature_id":"84c6dd2f539442a735ddad2dbf44c1744086e075f67b8f3edb677f60e9a862b7","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T00:06:48Z\n Ended: 2026-09-25T00:12:51Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4729344-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4729344-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103252727629877248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7123,"pr_state":"closed","pr_sha":"c07c99eb845db32bdffea8c5c152b60fa095f319","final_merged_sha":"c07c99eb845db32bdffea8c5c152b60fa095f319","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T22:38:21Z","started_at":"2026-09-24T22:38:21Z","completed_at":"2026-09-25T00:58:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T01:10:56.365736195Z","timing_metadata_checked_at":"2026-09-25T01:10:56.365736195Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T01:10:56.369549783Z","region_metadata_checked_at":"2026-09-25T01:10:56.369549783Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"267afb5f3b9ca3d17f1ea31fe16cf4db5921f0f99d29cba1f395496414e5b4be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103252727629877248","occurred_at":"2026-09-24T22:38:21Z","failed_at":"alert","signature_id":"00ca63290ee1dd8cd9cc17fb467e5abff9ca540cb77527300f636b61f37a158b","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-25T00:08:21Z\n Ended: 2026-09-25T00:18:21Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9877248-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-7487b8fc78-kvh8n\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-7487b8fc78-kvh8n (namespace kube-applier) on cluster ci01-j9877248-mgmt-1 has averaged 133.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"3c507a4f05fba20efc918e4d58965332ae4c6f373c57c67232a6ce37cc105f2f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103252727629877248","occurred_at":"2026-09-24T22:38:21Z","failed_at":"e2e","signature_id":"1074b07740a45107502c1b1a3cd307b20f68100d4367f76d22ba40ec484f0d3d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2nj474\" for cluster \"cluster-candidate-5-0-2nj474\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b7a380\u003e: \n failed to create NodePool np-2nj474: failed waiting for nodepool=\"np-2nj474\" for cluster \"cluster-candidate-5-0-2nj474\" in resourcegroup=\"rg-candidate-5-0-vjxqrms5jclz\" to finish creating: GET https://rp.j9877248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/514bc0c8-a828-4772-95da-f95954e05701\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/514bc0c8-a828-4772-95da-f95954e05701\",\n \"name\": \"514bc0c8-a828-4772-95da-f95954e05701\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T23:48:54.785691346Z\",\n \"endTime\": \"2026-09-25T00:08:00.18158216Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np to exist\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np to exist\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2nj474\" for cluster \"cluster-candidate-5-0-2nj474\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b7a380\u003e: \n failed to create NodePool np-2nj474: failed waiting for nodepool=\"np-2nj474\" for cluster \"cluster-candidate-5-0-2nj474\" in resourcegroup=\"rg-candidate-5-0-vjxqrms5jclz\" to finish creating: GET https://rp.j9877248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/514bc0c8-a828-4772-95da-f95954e05701\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/514bc0c8-a828-4772-95da-f95954e05701\",\n \"name\": \"514bc0c8-a828-4772-95da-f95954e05701\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T23:48:54.785691346Z\",\n \"endTime\": \"2026-09-25T00:08:00.18158216Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np to exist\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np to exist\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"64bc5ac277db781b54ff950d9ea3c0c424d37517be1bdac9b31411f402f5258f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","64bc5ac277db781b54ff950d9ea3c0c424d37517be1bdac9b31411f402f5258f"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103252721661382656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6821,"pr_state":"closed","pr_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","final_merged_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T22:38:19Z","started_at":"2026-09-24T22:38:19Z","completed_at":"2026-09-24T23:41:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T23:46:29.59696863Z","timing_metadata_checked_at":"2026-09-24T23:46:29.59696863Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T23:46:29.600286419Z","region_metadata_checked_at":"2026-09-24T23:46:29.600286419Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"bf0256c4e71eb785be8cb8085f1d95834b5fdfb999bafb0f745b12ee8718e754","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103252721661382656","occurred_at":"2026-09-24T22:38:19Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-24T23:03:18.918Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T23:03:20.600Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T23:03:21.313Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=d81bee64ce02324faba35ebcfefbaa384d773cf4fc43c777c70f8e291440954b portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1382656%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fd81bee64ce02324faba35ebcfefbaa384d773cf4fc43c777c70f8e291440954b\ntime=2026-09-24T23:05:52.100Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1382656/providers/Microsoft.Resources/deployments/d81bee64ce02324faba35ebcfefbaa384d773cf4fc43c777c70f8e291440954b/operationStatuses/08584113158845641130\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1382656/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1382656/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1382656\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 1a3e1a3d-0894-450c-b8f5-f4aae39286df:9/24/2026 11:04:34 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T23:07:52.108Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T23:07:53.036Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T23:07:53.967Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=e77ffa30236038ba6be6aa6728547670b7fd78c60b95d6556ff96ad801ee7bf4 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1382656%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fe77ffa30236038ba6be6aa6728547670b7fd78c60b95d6556ff96ad801ee7bf4\ntime=2026-09-24T23:24:01.808Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1382656/providers/Microsoft.Resources/deployments/e77ffa30236038ba6be6aa6728547670b7fd78c60b95d6556ff96ad801ee7bf4/operationStatuses/08584113156121186022\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1382656/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1382656/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1382656\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report f12586db-dfe7-4f9a-a86d-256d3a24ca8b:9/24/2026 11:23:57 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T23:26:01.815Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T23:26:02.592Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T23:26:03.555Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=8cd82016b5e13f611cfcadf8f8916eda701f48b5be5c35226e79dda3dd59a43e portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1382656%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F8cd82016b5e13f611cfcadf8f8916eda701f48b5be5c35226e79dda3dd59a43e\ntime=2026-09-24T23:33:18.520Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7146/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103249876442681344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7146,"pr_state":"closed","pr_sha":"210214039c8925fdc446fab903f1f8d1a21e74ae","final_merged_sha":"34a27988a2da6ca8b624fbab48af075e636b7eb6","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T22:27:01Z","started_at":"2026-09-24T22:27:01Z","completed_at":"2026-09-24T22:55:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T23:04:13.64116427Z","timing_metadata_checked_at":"2026-09-24T23:04:13.64116427Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T23:04:13.644568759Z","region_metadata_checked_at":"2026-09-24T23:04:13.644568759Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"1fd80d5f13d361cfb6c60856850c0dd3c932bd2a44f5eeba78db679b0d66a148","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7146/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103249876442681344","occurred_at":"2026-09-24T22:27:01Z","failed_at":"provision","signature_id":"8b0f84d3fd13e52b773c2d5762e6e413e0d4ff59cb390c1fe435499d6e4b2b92","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/kusto/maestroMqttDataConnection","test_suite":"templatize-pipeline","failure_text":"time=2026-09-24T22:47:58.457Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection description=\"Step maestroMqttDataConnection\\n Kind: ARM\\n Template: templates/maestro-mqtt-data-connection.bicep\\n Parameters: configurations/maestro-mqtt-data-connection.tmpl.bicepparam\"\ntime=2026-09-24T22:47:59.030Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection\ntime=2026-09-24T22:47:59.240Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection err=\"failed to run ARM step: failed to create deployment: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-kusto-us/providers/Microsoft.Resources/deployments/754de7cea577bd27c73251066594765f388be9580303d6f77fd3975785c55d9b\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidTemplate\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidTemplate\\\",\\n \\\"message\\\": \\\"Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'maestroMqttEventHubId'. The parameters defined in the template are: 'databaseName, eventHubId, eventhubEnabled, kustoConsumerGroupName, kustoDataConnectionName, kustoEnabled, kustoName, location'. Please see https://aka.ms/arm-pass-parameter-values for usage details.'.\\\",\\n \\\"additionalInfo\\\": [\\n {\\n \\\"type\\\": \\\"TemplateViolation\\\",\\n \\\"info\\\": {\\n \\\"lineNumber\\\": 0,\\n \\\"linePosition\\\": 0,\\n \\\"path\\\": \\\"\\\"\\n }\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0905a6f0720326490c59375e0404554c58e1e0996cc236e2a705a956863de47f","failure_pattern":"ERROR CODE: InvalidTemplate; detail message Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'maestroMqttEventHubId'. The parameters defined in the template are: 'databaseName, eventHubId, eventhubEnabled, kustoConsumerGroupName, kustoDataConnectionName, kustoEnabled, kustoName, location'.","search_query":"ERROR CODE: InvalidTemplate"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0905a6f0720326490c59375e0404554c58e1e0996cc236e2a705a956863de47f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103247783359156224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7148,"pr_state":"closed","pr_sha":"c3e1a8ef598ecb7f85e0ae9a62013997e2cf959c","final_merged_sha":"c3e1a8ef598ecb7f85e0ae9a62013997e2cf959c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T22:18:40Z","started_at":"2026-09-24T22:18:40Z","completed_at":"2026-09-25T00:43:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T00:49:57.944846362Z","timing_metadata_checked_at":"2026-09-25T00:49:57.944846362Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T00:49:57.952357338Z","region_metadata_checked_at":"2026-09-25T00:49:57.952357338Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"a62f43163ba2ca178dcebe697a164e0cde67fb65fcd7480bb3fdcd6316fa8cf7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103247783359156224","occurred_at":"2026-09-24T22:18:40Z","failed_at":"e2e","signature_id":"d790667a42bc853d57fab4f94768ab9d93901fef5701562ee2de998ed5c7396d","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:172]: failed to resolve test runner public IP\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d70460\u003e: \n failed to query public IP echo service: Get \"https://checkip.amazonaws.com\": net/http: TLS handshake timeout\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:172]: failed to resolve test runner public IP\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d70460\u003e: \n failed to query public IP echo service: Get \"https://checkip.amazonaws.com\": net/http: TLS handshake timeout\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2553fa7889a9af1fa3b8eb97a5f22ebd02ed4cb66f230b2442c0efe51892094d","failure_pattern":"failed to query public IP echo service: Get \"\u003curl\u003e\": net/http: TLS handshake timeout","search_query":"failed to query public IP echo service: Get \"https://checkip.amazonaws.com\": net/http: TLS handshake timeout"}},{"row_id":"c63e4390330ce53eddc5b00c3c243915c1a5ad2bbfc83aec84b01ea9171daf8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103247783359156224","occurred_at":"2026-09-24T22:18:40Z","failed_at":"alert","signature_id":"31141dc7fe43f6d2aa89337a0a716ccfb12d368d6474df9ae0a1733b03fcf72c","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T23:55:38Z\n Ended: 2026-09-25T00:01:38Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9156224-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-75497856bf-g2n4k\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-75497856bf-g2n4k (namespace kube-applier) on cluster ci01-j9156224-mgmt-1 has averaged 122.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","2553fa7889a9af1fa3b8eb97a5f22ebd02ed4cb66f230b2442c0efe51892094d"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7149/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103246251251208192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7149,"pr_state":"closed","pr_sha":"75f4b3d73fbc760e8b69df0db4067b8702fb2556","final_merged_sha":"75f4b3d73fbc760e8b69df0db4067b8702fb2556","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T22:12:37Z","started_at":"2026-09-24T22:12:37Z","completed_at":"2026-09-24T23:24:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T23:35:55.202990716Z","timing_metadata_checked_at":"2026-09-24T23:35:55.202990716Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T23:35:55.206799104Z","region_metadata_checked_at":"2026-09-24T23:35:55.206799104Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"7329298cff05ca89ef74d5d09a1f53706797fd0272eb5ed804181b21c3ad9992","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7149/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103246251251208192","occurred_at":"2026-09-24T22:12:37Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-24T22:47:35.665Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T22:47:37.519Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T22:47:38.378Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=308b140a9f4788d0c71596e1f1c6407e494b954ccb82eec993ef99dd6c349204 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1208192%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F308b140a9f4788d0c71596e1f1c6407e494b954ccb82eec993ef99dd6c349204\ntime=2026-09-24T22:50:09.688Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1208192/providers/Microsoft.Resources/deployments/308b140a9f4788d0c71596e1f1c6407e494b954ccb82eec993ef99dd6c349204/operationStatuses/08584113168276968799\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1208192/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1208192/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1208192\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report f8b34aa2-2709-4f1c-9984-1cc5067ced73:9/24/2026 10:49:04 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T22:52:09.690Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T22:52:10.439Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T22:52:11.338Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=51e0669f32303523a85a7a3544429ef9986c68c681e61ebd68b5aed22dbaa254 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1208192%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F51e0669f32303523a85a7a3544429ef9986c68c681e61ebd68b5aed22dbaa254\ntime=2026-09-24T23:08:15.812Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1208192/providers/Microsoft.Resources/deployments/51e0669f32303523a85a7a3544429ef9986c68c681e61ebd68b5aed22dbaa254/operationStatuses/08584113165546052729\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1208192/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1208192/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1208192\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report ffdd7810-797d-49fb-b16c-e433118bbda7:9/24/2026 11:07:35 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T23:10:15.814Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T23:10:16.691Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T23:10:17.403Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=8ca84a4ab65a2a0c93ac0952e362ffa1c5ea7a1acc5faf5e342778682e18bace portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1208192%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F8ca84a4ab65a2a0c93ac0952e362ffa1c5ea7a1acc5faf5e342778682e18bace\ntime=2026-09-24T23:17:35.285Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7059/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103245132269621248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7059,"pr_state":"closed","pr_sha":"b22cddf8453a7f89c54cd40aafd6d1bcecefc802","final_merged_sha":"b22cddf8453a7f89c54cd40aafd6d1bcecefc802","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T22:08:10Z","started_at":"2026-09-24T22:08:10Z","completed_at":"2026-09-25T00:50:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T01:00:43.474007822Z","timing_metadata_checked_at":"2026-09-25T01:00:43.474007822Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T01:00:43.477391411Z","region_metadata_checked_at":"2026-09-25T01:00:43.477391411Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"aab9ad5846d1df703c9073d69c5d1f18974de7233447b4c50a764b188c9de89f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7059/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103245132269621248","occurred_at":"2026-09-24T22:08:10Z","failed_at":"alert","signature_id":"cdf5e1de46d904b5754f4fcc48d70aa65ef0855075ecd7f8c6dfc039233eac7c","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T23:56:24Z\n Ended: 2026-09-25T00:07:26Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9621248-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-84ccbd4674-cnjk2\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-84ccbd4674-cnjk2 (namespace kube-applier) on cluster ci01-j9621248-mgmt-1 has averaged 153.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233926699945984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7135,"pr_state":"closed","pr_sha":"78d2e7cdedac95816c4452d91dfcae35ac93f66a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T21:23:38Z","started_at":"2026-09-24T21:23:38Z","completed_at":"2026-09-24T23:48:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T23:57:09.814332797Z","timing_metadata_checked_at":"2026-09-24T23:57:09.814332797Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T23:57:09.819372883Z","region_metadata_checked_at":"2026-09-24T23:57:09.819372883Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"227ba1e6eeecf96884cc240f7f302acc81032b4087f5719b909cd6b67f862755","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233926699945984","occurred_at":"2026-09-24T21:23:38Z","failed_at":"e2e","signature_id":"142775d0e7b84904771b9c9baeca0e991ed25c62693ee1753f67f0e8192808e1","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc00145fb18\u003e: \n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-4bnpzclpb4vn\" to finish creating: GET https://rp.j9945984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ae2ff5d-353f-4924-8074-8541fe3fffdf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ae2ff5d-353f-4924-8074-8541fe3fffdf\",\n \"name\": \"1ae2ff5d-353f-4924-8074-8541fe3fffdf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T22:34:11.896891667Z\",\n \"endTime\": \"2026-09-24T22:53:20.096362323Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-vbqjx-klzb2: Failed: virtualmachine failed to create or update. err: failed to get existing resource feature-aggregation-4bnpzclpb4vn--managed/agg-cluster-agg-np-a-vbqjx-klzb2 (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-4bnpzclpb4vn--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-vbqjx-klzb2\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '07aa0e83-353e-444c-9f1d-d023ac3c5396' exceeded. Please try again after '10' seconds after additional tokens are available. ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc00145fb18\u003e: \n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-4bnpzclpb4vn\" to finish creating: GET https://rp.j9945984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ae2ff5d-353f-4924-8074-8541fe3fffdf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ae2ff5d-353f-4924-8074-8541fe3fffdf\",\n \"name\": \"1ae2ff5d-353f-4924-8074-8541fe3fffdf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T22:34:11.896891667Z\",\n \"endTime\": \"2026-09-24T22:53:20.096362323Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-vbqjx-klzb2: Failed: virtualmachine failed to create or update. err: failed to get existing resource feature-aggregation-4bnpzclpb4vn--managed/agg-cluster-agg-np-a-vbqjx-klzb2 (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-4bnpzclpb4vn--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-vbqjx-klzb2\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '07aa0e83-353e-444c-9f1d-d023ac3c5396' exceeded. Please try again after '10' seconds after additional tokens are available. ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"372d0c16d6516fbc2516124c1f8eb533277ffb462595208eacf857f749c90520","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. ... message truncated; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"48bd7380e355105619cd7a25d92e5ccca710f02d6dad4a2aaba9b4b8c5198b50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233926699945984","occurred_at":"2026-09-24T21:23:38Z","failed_at":"alert","signature_id":"6a245a5382730698531a737265dac078b3a944a08cb0f737e3cfdc39f19615db","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T22:55:27Z\n Ended: 2026-09-24T23:06:30Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9945984-mgmt-2\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-76c7b488dc-tnbsr\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-76c7b488dc-tnbsr (namespace kube-applier) on cluster ci01-j9945984-mgmt-2 has averaged 135.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"a7febdf04da4d4edd8f412d2b168854011f035917f0b6f4730bd8c3ad36c60a2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233926699945984","occurred_at":"2026-09-24T21:23:38Z","failed_at":"alert","signature_id":"d8855120e86c8409ebf1202c522f0520dd727db9ff94a1dfff81d2c5c15fcc43","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T22:56:37Z\n Ended: 2026-09-24T23:04:39Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9945984-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9945984-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"b8999e83b8c2f638d1af7a7deed8c9529e2a313bc0415f29fddc6c04621ce69b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233926699945984","occurred_at":"2026-09-24T21:23:38Z","failed_at":"e2e","signature_id":"84b235bf896d01e3deaa66856a2e9786768f1d93481789804dd05b50eee4738a","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc003790c60\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-2tslxd58w56h\" to finish creating: GET https://rp.j9945984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3bbdcadb-3a21-4369-a3b9-263a4eb93a49\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3bbdcadb-3a21-4369-a3b9-263a4eb93a49\",\n \"name\": \"3bbdcadb-3a21-4369-a3b9-263a4eb93a49\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T22:33:47.939426596Z\",\n \"endTime\": \"2026-09-24T22:52:50.218624914Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j9h7b2t4y3p0g4b-np-delete-main-vngst-vfmpn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-2tslxd58w56h--managed/providers/Microsoft.Compute/virtualMachines/j9h7b2t4y3p0g4b-np-delete-main-vngst-vfmpn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j9h7b2t4y3p0g4b-np-delete-main-vngst-vfmpn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-2tslxd58w56h--managed/providers/Microsoft.Compute/virtualMachines/j9h7b2t4y3p0g4b-np-delete-main-vngst-vfmpn to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc003790c60\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-2tslxd58w56h\" to finish creating: GET https://rp.j9945984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3bbdcadb-3a21-4369-a3b9-263a4eb93a49\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3bbdcadb-3a21-4369-a3b9-263a4eb93a49\",\n \"name\": \"3bbdcadb-3a21-4369-a3b9-263a4eb93a49\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T22:33:47.939426596Z\",\n \"endTime\": \"2026-09-24T22:52:50.218624914Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j9h7b2t4y3p0g4b-np-delete-main-vngst-vfmpn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-2tslxd58w56h--managed/providers/Microsoft.Compute/virtualMachines/j9h7b2t4y3p0g4b-np-delete-main-vngst-vfmpn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j9h7b2t4y3p0g4b-np-delete-main-vngst-vfmpn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-2tslxd58w56h--managed/providers/Microsoft.Compute/virtualMachines/j9h7b2t4y3p0g4b-np-delete-main-vngst-vfmpn to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-2tslxd58w56h\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","372d0c16d6516fbc2516124c1f8eb533277ffb462595208eacf857f749c90520","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233919481548800","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7009,"pr_state":"closed","pr_sha":"30ab7aab2c6cd7fcf1f6520d0a43a948520ff9af","final_merged_sha":"c83bdabaf08bdb32938b4526f3e12a94c26f811c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T21:23:37Z","started_at":"2026-09-24T21:23:37Z","completed_at":"2026-09-24T23:39:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T23:46:29.781688137Z","timing_metadata_checked_at":"2026-09-24T23:46:29.781688137Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T23:46:29.784378429Z","region_metadata_checked_at":"2026-09-24T23:46:29.784378429Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"0bd00931f03fa89f8c12db02eebadf9e7c088f20648399fa5008c45ba2481bf2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233919481548800","occurred_at":"2026-09-24T21:23:37Z","failed_at":"alert","signature_id":"8d1d5c509bd74b9cfd90fd52435b1d0a29f7a99c0e5bc532798559b82949daa7","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T22:46:42Z\n Ended: 2026-09-24T23:00:44Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j1548800-mgmt-2\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-76f9547896-tzmvx\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-76f9547896-tzmvx (namespace kube-applier) on cluster ci01-j1548800-mgmt-2 has averaged 139.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"5c1caa050433760237fbae637a5be420fbbab358bb7e721312e304f9295bc887","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233919481548800","occurred_at":"2026-09-24T21:23:37Z","failed_at":"e2e","signature_id":"9783002e2eb58bbef4bfb807305e4c98f54648869ece93a50ed71011664015e3","test_name":"KSM HCP Metrics metrics should be present in HCP Azure Monitoring Workspace","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:185]: Timed out after 1500.005s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:183 with:\nexpected kube_node_status_condition from kube-state-metrics; kube_node_info from kube-state-metrics but got no results\nExpected\n \u003c[]string | len:2, cap:2\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:185]: Timed out after 1500.005s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:183 with:\nexpected kube_node_status_condition from kube-state-metrics; kube_node_info from kube-state-metrics but got no results\nExpected\n \u003c[]string | len:2, cap:2\u003e: ...\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d91796eeda5752e6e2b49b398ec435f90a4ab6d3394ea2d0cd4ce5b2c2f30a11","failure_pattern":"not all KSM metrics appeared in Azure Monitor","search_query":"not all KSM metrics appeared in Azure Monitor"}},{"row_id":"c84f37de35b1777011786b02b26a546baccf6812db501802ab9a43a1cb8c15ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233919481548800","occurred_at":"2026-09-24T21:23:37Z","failed_at":"alert","signature_id":"d06adfd8f5d6c7ecc9b42d9b62223e573b7f16b0df6ea81be882241614b5334c","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T22:49:18Z\n Ended: 2026-09-24T22:55:14Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1548800-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1548800-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","d91796eeda5752e6e2b49b398ec435f90a4ab6d3394ea2d0cd4ce5b2c2f30a11","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233417079427072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7068,"pr_state":"closed","pr_sha":"5d483559deea65ed903f146fc4f20cbbc522559d","final_merged_sha":"269e5d56bcfac6dc9f8d3ca87b7fb1ea84bf53b2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T21:21:37Z","started_at":"2026-09-24T21:21:37Z","completed_at":"2026-09-24T23:53:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T00:08:02.067219238Z","timing_metadata_checked_at":"2026-09-25T00:08:02.067219238Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T00:08:02.069858431Z","region_metadata_checked_at":"2026-09-25T00:08:02.069858431Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"fcace8e5689d195c80b5e82cace68952e51e570ba9ef20bf6b58c0287ddaefed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233417079427072","occurred_at":"2026-09-24T21:21:37Z","failed_at":"alert","signature_id":"cd75991f1a2d3f936234a24c188b3b95270f16a178b722a2c9be2086d6903383","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T22:57:46Z\n Ended: 2026-09-24T23:04:47Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9427072-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9427072-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"fdcd264b3f002b25cde0b3a449fb7cc571d7258181800990a1ed3f753c34b3a8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233417079427072","occurred_at":"2026-09-24T21:21:37Z","failed_at":"alert","signature_id":"cedddb9dfe24b0472f0b69938e215469ad1fa1732808690eb588c2842038524b","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T22:54:35Z\n Ended: 2026-09-24T23:07:36Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9427072-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-67dcb56ffb-j558w\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-67dcb56ffb-j558w (namespace kube-applier) on cluster ci01-j9427072-mgmt-1 has averaged 152.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103232602474287104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6359,"pr_state":"open","pr_sha":"43b35a36599a28c88038c301af1400949448f0bf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T21:18:23Z","started_at":"2026-09-24T21:18:23Z","completed_at":"2026-09-24T22:21:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T22:33:06.642820543Z","timing_metadata_checked_at":"2026-09-24T22:33:06.642820543Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T22:33:06.647453028Z","region_metadata_checked_at":"2026-09-24T22:33:06.647453028Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c0708e41c969ca89c060ee11d647b4557fda03f53514114d0b00736fe7cdac16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103232602474287104","occurred_at":"2026-09-24T21:18:23Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-24T21:46:20.785Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T21:46:22.588Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T21:46:23.803Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=ec05559300234de72b80a6e22709f4ef34d43f9f8dca190a03f18c6598f87ab3 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4287104%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fec05559300234de72b80a6e22709f4ef34d43f9f8dca190a03f18c6598f87ab3\ntime=2026-09-24T21:48:54.587Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4287104/providers/Microsoft.Resources/deployments/ec05559300234de72b80a6e22709f4ef34d43f9f8dca190a03f18c6598f87ab3/operationStatuses/08584113205023610599\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4287104/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4287104/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j4287104\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 0ab9149a-94e0-4a65-8802-537432f2290e:9/24/2026 9:47:43 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T21:50:54.588Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T21:50:55.433Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T21:50:56.536Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9d83a8d4dd91cbf68ead841f4207df9ed601aadefa137376ddb8486d026e9747 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4287104%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9d83a8d4dd91cbf68ead841f4207df9ed601aadefa137376ddb8486d026e9747\ntime=2026-09-24T22:07:00.971Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4287104/providers/Microsoft.Resources/deployments/9d83a8d4dd91cbf68ead841f4207df9ed601aadefa137376ddb8486d026e9747/operationStatuses/08584113202295374363\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4287104/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4287104/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j4287104\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report a8c3ddb4-7313-4f9c-afe7-f7cd44faff1d:9/24/2026 10:06:20 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T22:09:00.977Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T22:09:02.023Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T22:09:02.790Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=1387cc5c3bd9b4e7f11e36b52457e789e5c0ac38a7e1b0185ed2b0d76dfc6b3f portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4287104%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F1387cc5c3bd9b4e7f11e36b52457e789e5c0ac38a7e1b0185ed2b0d76dfc6b3f\ntime=2026-09-24T22:16:20.434Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103224244262408192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"89d7db87442ead3f3e8fcb09b59a5aaf74769ffd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T20:45:10Z","started_at":"2026-09-24T20:45:10Z","completed_at":"2026-09-24T22:55:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T23:04:12.105885652Z","timing_metadata_checked_at":"2026-09-24T23:04:12.105885652Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T23:04:12.10967214Z","region_metadata_checked_at":"2026-09-24T23:04:12.10967214Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"9a080f9d1ea197cd63a27b2a310655c6af735926078c97174bcbab40b77c706d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103224244262408192","occurred_at":"2026-09-24T20:45:10Z","failed_at":"e2e","signature_id":"ecbec1ab9f9617f6b0594d4fbd470c36978a735ccd4e20c469402d74bd7943c7","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.003s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001ea9b70\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.003s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001ea9b70\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103218872113958912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6993,"pr_state":"closed","pr_sha":"54fa83b6be2bda741a3e467491a8b8caf8a2dbe3","final_merged_sha":"1a558ec553718fc2df21030c08516ca31075af74","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T20:23:49Z","started_at":"2026-09-24T20:23:49Z","completed_at":"2026-09-24T22:56:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T23:04:12.746143074Z","timing_metadata_checked_at":"2026-09-24T23:04:12.746143074Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T23:04:12.748820566Z","region_metadata_checked_at":"2026-09-24T23:04:12.748820566Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"c02457383a35c216a1d66793fea1f15361b9b16aba10fa0ba2ec26b4f5d9b74b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103218872113958912","occurred_at":"2026-09-24T20:23:49Z","failed_at":"alert","signature_id":"f448f1107e13285139f08f0b5d64b030989f1fce9c96e0627368c8e075d775c1","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T22:06:27Z\n Ended: 2026-09-24T22:17:26Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3958912-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-5fc577f88f-scc46\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-5fc577f88f-scc46 (namespace mgmt-agent) on cluster ci01-j3958912-mgmt-1 has averaged 144.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-24T22:06:27Z\n Ended: 2026-09-24T22:24:29Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3958912-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-587bd64cb5-zm7df\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-587bd64cb5-zm7df (namespace kube-applier) on cluster ci01-j3958912-mgmt-1 has averaged 192.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103218868364251136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"closed","pr_sha":"87e40a07142a1457ed3f83374cc5d085c87d8f29","final_merged_sha":"e772682c6ada7bf3fd5674946d72fe25f79d834b","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-24T20:23:48Z","started_at":"2026-09-24T20:23:48Z","completed_at":"2026-09-24T23:00:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T23:04:12.361017924Z","timing_metadata_checked_at":"2026-09-24T23:04:12.361017924Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T23:04:12.363877215Z","region_metadata_checked_at":"2026-09-24T23:04:12.363877215Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103218827738222592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7045,"pr_state":"closed","pr_sha":"711d134f86fa2910df2819fd4922f71db33ae23c","final_merged_sha":"711d134f86fa2910df2819fd4922f71db33ae23c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-24T20:23:36Z","started_at":"2026-09-24T20:23:36Z","completed_at":"2026-09-24T23:14:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T23:25:07.101680508Z","timing_metadata_checked_at":"2026-09-24T23:25:07.101680508Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T23:25:07.118267356Z","region_metadata_checked_at":"2026-09-24T23:25:07.118267356Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103218829420138496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7053,"pr_state":"closed","pr_sha":"b938b99cc7c5a09bd230dfa6d470fb9e6a8d6e0a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T20:23:36Z","started_at":"2026-09-24T20:23:36Z","completed_at":"2026-09-24T22:53:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T23:04:13.158422837Z","timing_metadata_checked_at":"2026-09-24T23:04:13.158422837Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T23:04:13.161669526Z","region_metadata_checked_at":"2026-09-24T23:04:13.161669526Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"23e576320d8608cbe5695f885619995ece8d363633390b8b3ab92c7fb980dc39","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103218829420138496","occurred_at":"2026-09-24T20:23:36Z","failed_at":"alert","signature_id":"09e1889769e13cbac0f19e0bdd92a9ef75986032637b4b9ddaaed3dbdd08afff","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T21:32:55Z\n Ended: 2026-09-24T22:06:57Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0138496-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0138496-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103213571209957376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6969,"pr_state":"closed","pr_sha":"c2d193b5476ca0161cb6c191d698b62286404a95","final_merged_sha":"6dc92c0e1eae82818633321751269a6063e49fa2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T20:02:45Z","started_at":"2026-09-24T20:02:45Z","completed_at":"2026-09-24T20:58:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T21:08:42.296087605Z","timing_metadata_checked_at":"2026-09-24T21:08:42.296087605Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T21:08:42.299217694Z","region_metadata_checked_at":"2026-09-24T21:08:42.299217694Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"e67e18622029cd3824733dedc9395728c79cc0d2074c553eb1fcb4af85d6f017","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103213571209957376","occurred_at":"2026-09-24T20:02:45Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-24T20:18:09.372Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T20:18:10.737Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T20:18:11.463Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=268f9a48194bf4930329bdee16efbcd5d7512de97a83dc8f6bcc7f38d72db0d3 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9957376%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F268f9a48194bf4930329bdee16efbcd5d7512de97a83dc8f6bcc7f38d72db0d3\ntime=2026-09-24T20:20:41.992Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9957376/providers/Microsoft.Resources/deployments/268f9a48194bf4930329bdee16efbcd5d7512de97a83dc8f6bcc7f38d72db0d3/operationStatuses/08584113257944466304\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9957376/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9957376/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j9957376\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 89fa80c9-7dfa-40a9-b9d6-63fcb2f0a170:9/24/2026 8:19:22 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T20:22:41.996Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T20:22:42.837Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T20:22:44.002Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=6f3210d741cfc8b54935dc00f46515c00ade6d74e7c58e9bf6ba7d1aab0c81f4 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9957376%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6f3210d741cfc8b54935dc00f46515c00ade6d74e7c58e9bf6ba7d1aab0c81f4\ntime=2026-09-24T20:38:17.608Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9957376/providers/Microsoft.Resources/deployments/6f3210d741cfc8b54935dc00f46515c00ade6d74e7c58e9bf6ba7d1aab0c81f4/operationStatuses/08584113255222176724\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9957376/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9957376/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j9957376\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 4faefff8-fb62-4a6e-a46a-092208de93bd:9/24/2026 8:38:06 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T20:40:17.609Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T20:40:18.242Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T20:40:19.164Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=8a402905cd599ad1785df05f428ddfa7215782e267e626604f9d4518b1ef1ac6 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9957376%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F8a402905cd599ad1785df05f428ddfa7215782e267e626604f9d4518b1ef1ac6\ntime=2026-09-24T20:48:09.044Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103205737139277824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7142,"pr_state":"closed","pr_sha":"cfda26d7698257f0afa1b25f95e39ab26e8f37ef","final_merged_sha":"cfda26d7698257f0afa1b25f95e39ab26e8f37ef","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T19:31:38Z","started_at":"2026-09-24T19:31:38Z","completed_at":"2026-09-24T20:24:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T20:37:01.035056036Z","timing_metadata_checked_at":"2026-09-24T20:37:01.035056036Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T20:37:01.038797023Z","region_metadata_checked_at":"2026-09-24T20:37:01.038797023Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ab926cbae04cb589c22a8cddb94d0049c24a7215e3ea4fff907cc02d972423a2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103205737139277824","occurred_at":"2026-09-24T19:31:38Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-24T19:48:10.021Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T19:48:11.681Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T19:48:12.351Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=8386e8d508ae47a173f0f0f70d0cd66da073fbc1943f414f1f9759f57da1762c portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9277824%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F8386e8d508ae47a173f0f0f70d0cd66da073fbc1943f414f1f9759f57da1762c\ntime=2026-09-24T19:50:43.107Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9277824/providers/Microsoft.Resources/deployments/8386e8d508ae47a173f0f0f70d0cd66da073fbc1943f414f1f9759f57da1762c/operationStatuses/08584113275935641416\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9277824/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9277824/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j9277824\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report b4253bc8-c855-4017-a23b-60efdfb738e7:9/24/2026 7:49:19 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T19:52:43.108Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T19:52:43.925Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T19:52:44.692Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=cffa80b2bdeace3fdd2e49987efcb77e91106c320dc4d1d1d231adf945f08655 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9277824%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fcffa80b2bdeace3fdd2e49987efcb77e91106c320dc4d1d1d231adf945f08655\ntime=2026-09-24T20:08:18.722Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9277824/providers/Microsoft.Resources/deployments/cffa80b2bdeace3fdd2e49987efcb77e91106c320dc4d1d1d231adf945f08655/operationStatuses/08584113273212531078\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9277824/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9277824/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j9277824\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report e73d26b8-59e2-413d-a6be-119be1a2cca3:9/24/2026 8:07:51 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T20:10:18.723Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T20:10:19.614Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T20:10:20.734Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=f0fc5c476fd609dff09d4a0b97983314608643509592991a398ebc058ae006b9 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9277824%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff0fc5c476fd609dff09d4a0b97983314608643509592991a398ebc058ae006b9\ntime=2026-09-24T20:18:09.660Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103203575923478528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"open","pr_sha":"e8380b1c35d0196ffaedbde86e8c22c316ddfb1a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-24T19:22:55Z","started_at":"2026-09-24T19:22:55Z","completed_at":"2026-09-24T21:35:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T21:40:56.327643857Z","timing_metadata_checked_at":"2026-09-24T21:40:56.327643857Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T21:40:56.330214548Z","region_metadata_checked_at":"2026-09-24T21:40:56.330214548Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103203513864556544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6970,"pr_state":"closed","pr_sha":"90c3b8a82bebade41bd21c89202388c54efd1a17","final_merged_sha":"af55f114edf1b1d398480eeee6a41ae706e821e2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T19:22:46Z","started_at":"2026-09-24T19:22:46Z","completed_at":"2026-09-24T21:43:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T21:51:20.483211403Z","timing_metadata_checked_at":"2026-09-24T21:51:20.483211403Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T21:51:20.485973294Z","region_metadata_checked_at":"2026-09-24T21:51:20.485973294Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"51adfdf403b7a770e5aed561b281d27aed90844c411c7aec4ace23286ea21539","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103203513864556544","occurred_at":"2026-09-24T19:22:46Z","failed_at":"e2e","signature_id":"2dd1d4adf03c4cf1ef2152b091a8d8db32bd11c08e0192a8877e14758cab2f4d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-c44pd8c2jtgp/cluster-candidate-4-21-k4rdfg should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010bc2e0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-k4rdfg: failed waiting for cluster=\"cluster-candidate-4-21-k4rdfg\" in resourcegroup=\"rg-candidate-4-21-c44pd8c2jtgp\" to finish creating: GET https://rp.j4556544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c4682523-7a01-4560-b35e-4fc4d15613ae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c4682523-7a01-4560-b35e-4fc4d15613ae\",\n \"name\": \"c4682523-7a01-4560-b35e-4fc4d15613ae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T20:22:03.886308121Z\",\n \"endTime\": \"2026-09-24T20:51:09.533160535Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 24m24s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-c44pd8c2jtgp/cluster-candidate-4-21-k4rdfg should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010bc2e0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-k4rdfg: failed waiting for cluster=\"cluster-candidate-4-21-k4rdfg\" in resourcegroup=\"rg-candidate-4-21-c44pd8c2jtgp\" to finish creating: GET https://rp.j4556544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c4682523-7a01-4560-b35e-4fc4d15613ae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c4682523-7a01-4560-b35e-4fc4d15613ae\",\n \"name\": \"c4682523-7a01-4560-b35e-4fc4d15613ae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T20:22:03.886308121Z\",\n \"endTime\": \"2026-09-24T20:51:09.533160535Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 24m24s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"76f1390ec81e91acba51c96ea39654ca03a2ca4a47bda9df50c2443d3c8d0b3b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103203513864556544","occurred_at":"2026-09-24T19:22:46Z","failed_at":"alert","signature_id":"9148700f82fd5b370a512210ca6a6cb677852f9f2d8abcc81bf3649ab5047f43","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T20:33:02Z\n Ended: 2026-09-24T21:02:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4556544-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t1jgr6a3uevc16juiqil4ndk5set1ce-o5o5w3j3g5c4j6m\", pod=\"router-66cb8fd48d-wgc7r\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t1jgr6a3uevc16juiqil4ndk5set1ce-o5o5w3j3g5c4j6m/router-66cb8fd48d-wgc7r has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"77ce6bdf552bc506108edaf38a4bc7a03ee3b37a39744f5893c336aa6410a6a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103203513864556544","occurred_at":"2026-09-24T19:22:46Z","failed_at":"e2e","signature_id":"073b0db0ceea3f28f404fe1b7904861cb57586c11302c87c1ccfd472e1f2a7a3","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b80880\u003e: \n failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-cpn7mwhqrdqw\" to finish creating: GET https://rp.j4556544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2d56b67b-43e0-4cd7-a969-5552b2f1c302\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2d56b67b-43e0-4cd7-a969-5552b2f1c302\",\n \"name\": \"2d56b67b-43e0-4cd7-a969-5552b2f1c302\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T20:50:39.494369799Z\",\n \"endTime\": \"2026-09-24T21:09:39.1222123Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-7rv28: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-drpbc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-7rv28: Creating: virtualmachine creating or updating\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-drpbc: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b80880\u003e: \n failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-cpn7mwhqrdqw\" to finish creating: GET https://rp.j4556544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2d56b67b-43e0-4cd7-a969-5552b2f1c302\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2d56b67b-43e0-4cd7-a969-5552b2f1c302\",\n \"name\": \"2d56b67b-43e0-4cd7-a969-5552b2f1c302\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T20:50:39.494369799Z\",\n \"endTime\": \"2026-09-24T21:09:39.1222123Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-7rv28: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-drpbc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-7rv28: Creating: virtualmachine creating or updating\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-drpbc: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b4593ef0dd9fbe369bd9dc5fb00d11d789b0f9fff11fea0544e9abdf8738d8b1","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID ; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8c9e0171b1efa09c0d3f92b29fcbc05c6f9eb12b952e51090f815e5ec09427d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103203513864556544","occurred_at":"2026-09-24T19:22:46Z","failed_at":"e2e","signature_id":"b24161097af08c2ace32acbd8380e530f701f116d3593a7a6af06880e3f2dbd7","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017d2100\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kmp4pw976cvp\" to finish creating: GET https://rp.j4556544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fe3f92f-008e-475a-a316-514fa9d56551\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fe3f92f-008e-475a-a316-514fa9d56551\",\n \"name\": \"4fe3f92f-008e-475a-a316-514fa9d56551\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T20:45:04.211205802Z\",\n \"endTime\": \"2026-09-24T21:04:09.101063234Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-kmp4pw976cvp--managed/providers/Microsoft.Compute/virtualMachines/l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-kmp4pw976cvp--managed/providers/Microsoft.Compute/virtualMachines/l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017d2100\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kmp4pw976cvp\" to finish creating: GET https://rp.j4556544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fe3f92f-008e-475a-a316-514fa9d56551\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fe3f92f-008e-475a-a316-514fa9d56551\",\n \"name\": \"4fe3f92f-008e-475a-a316-514fa9d56551\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T20:45:04.211205802Z\",\n \"endTime\": \"2026-09-24T21:04:09.101063234Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-kmp4pw976cvp--managed/providers/Microsoft.Compute/virtualMachines/l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-kmp4pw976cvp--managed/providers/Microsoft.Compute/virtualMachines/l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"881c8a92b161acdd9b8088fd7670ee486a4176a53d8400a1bad38528667ae5f4","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 2, want \u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"de8f026c4a9e8b9543f7b0027c72eceef770edaf35beeab6ec9d1400d7dc308f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103203513864556544","occurred_at":"2026-09-24T19:22:46Z","failed_at":"alert","signature_id":"474887e104d72f6b5e1fe9db5422a8d481f6d2378182349a4ce3015e81b513f6","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T20:54:54Z\n Ended: 2026-09-24T21:08:58Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4556544-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-5c8f6cff9-kg69g\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-5c8f6cff9-kg69g (namespace kube-applier) on cluster ci01-j4556544-mgmt-1 has averaged 151.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","881c8a92b161acdd9b8088fd7670ee486a4176a53d8400a1bad38528667ae5f4","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","b4593ef0dd9fbe369bd9dc5fb00d11d789b0f9fff11fea0544e9abdf8738d8b1"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6866/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103203214361890816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6866,"pr_state":"open","pr_sha":"d3c31a822d8049904889e6efe135f041de1e5ab9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T19:21:25Z","started_at":"2026-09-24T19:21:25Z","completed_at":"2026-09-24T21:40:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T21:51:20.116108847Z","timing_metadata_checked_at":"2026-09-24T21:51:20.116108847Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T21:51:20.121138231Z","region_metadata_checked_at":"2026-09-24T21:51:20.121138231Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"02da9f5c29e147b7c80c97b74f32ba309d82a929f50faed9fbfd18aed7252986","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6866/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103203214361890816","occurred_at":"2026-09-24T19:21:25Z","failed_at":"alert","signature_id":"855925e2df7033b23d85ecff7ede1ca6cb961907816303723c8e216cb000b37a","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T20:40:22Z\n Ended: 2026-09-24T20:55:25Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j1890816-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-64688fd65-565hs\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-64688fd65-565hs (namespace kube-applier) on cluster ci01-j1890816-mgmt-1 has averaged 140.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103200191157899264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7123,"pr_state":"closed","pr_sha":"c07c99eb845db32bdffea8c5c152b60fa095f319","final_merged_sha":"c07c99eb845db32bdffea8c5c152b60fa095f319","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T19:09:35Z","started_at":"2026-09-24T19:09:35Z","completed_at":"2026-09-24T21:25:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T21:29:56.215557621Z","timing_metadata_checked_at":"2026-09-24T21:29:56.215557621Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T21:29:56.220751104Z","region_metadata_checked_at":"2026-09-24T21:29:56.220751104Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"a511298f5f81f7223bf500f6017b8bcd9841bc2bfc265b291be8c4e8ceea1959","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103200191157899264","occurred_at":"2026-09-24T19:09:35Z","failed_at":"alert","signature_id":"784e6484d3fc29d5f43a3850fcf318afb800acb38164e1b18952339bbb4b08b9","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T20:27:31Z\n Ended: 2026-09-24T20:43:31Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7899264-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-7d9bb9cbf4-lx422\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-7d9bb9cbf4-lx422 (namespace kube-applier) on cluster ci01-j7899264-mgmt-1 has averaged 142.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103200143447691264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6821,"pr_state":"closed","pr_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","final_merged_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T19:09:24Z","started_at":"2026-09-24T19:09:24Z","completed_at":"2026-09-24T21:29:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T21:40:55.908214689Z","timing_metadata_checked_at":"2026-09-24T21:40:55.908214689Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T21:40:55.913352172Z","region_metadata_checked_at":"2026-09-24T21:40:55.913352172Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"31ad625b097354b94430e9e86d7bd07ef072d6d2ce1def60e8e01079b42c5674","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103200143447691264","occurred_at":"2026-09-24T19:09:24Z","failed_at":"alert","signature_id":"b7cba4cec6f5a81a201092312fdb1ca862649a7f4b0539eae14accd98ac8dba5","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T20:32:31Z\n Ended: 2026-09-24T20:42:34Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7691264-mgmt-2\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-656dd46fd9-5k7hr\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-656dd46fd9-5k7hr (namespace kube-applier) on cluster ci01-j7691264-mgmt-2 has averaged 133.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"c1aa5fe3897e3543ac7e3ec9fa796c65f2332c05c2245725c015994477d51452","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103200143447691264","occurred_at":"2026-09-24T19:09:24Z","failed_at":"e2e","signature_id":"5b25ef7df46295d6b0322467710d77611928e860a415a5c20e227a626a8235c7","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:209]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc00268e060\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:209]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc00268e060\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a87df562eaa8a0bfda8c14840c36e7197e35a3b92f22e4f536b1e3d2ed289ff","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","7a87df562eaa8a0bfda8c14840c36e7197e35a3b92f22e4f536b1e3d2ed289ff"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103198517097926656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6915,"pr_state":"open","pr_sha":"e207b76e733cb65c677c16c7f38706c45baff4cd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T19:02:55Z","started_at":"2026-09-24T19:02:55Z","completed_at":"2026-09-24T19:57:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T20:06:12.623191897Z","timing_metadata_checked_at":"2026-09-24T20:06:12.623191897Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T20:06:12.626406083Z","region_metadata_checked_at":"2026-09-24T20:06:12.626406083Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"11dfbb65ecf34472c36eda7a42ff6299fd6098e456967ed700ff17ab364e0def","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103198517097926656","occurred_at":"2026-09-24T19:02:55Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-24T19:20:36.650Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T19:20:38.348Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T19:20:39.321Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=8029ced0c895f25455a5b3aef3cb6587f727116d417e35bdf8fe9035696722ce portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7926656%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F8029ced0c895f25455a5b3aef3cb6587f727116d417e35bdf8fe9035696722ce\ntime=2026-09-24T19:23:40.053Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7926656/providers/Microsoft.Resources/deployments/8029ced0c895f25455a5b3aef3cb6587f727116d417e35bdf8fe9035696722ce/operationStatuses/08584113292468969733\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7926656/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7926656/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j7926656\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report a252f715-9fc8-4ac8-878f-1814dabfaa32:9/24/2026 7:21:50 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T19:25:40.054Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T19:25:40.873Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T19:25:41.584Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=c3ba64a3fe7ff1ec2c799bce029586074c1a0772905a319489e4ddaa14a5c296 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7926656%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fc3ba64a3fe7ff1ec2c799bce029586074c1a0772905a319489e4ddaa14a5c296\ntime=2026-09-24T19:41:14.858Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7926656/providers/Microsoft.Resources/deployments/c3ba64a3fe7ff1ec2c799bce029586074c1a0772905a319489e4ddaa14a5c296/operationStatuses/08584113289443397811\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7926656/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7926656/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j7926656\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 75f3e5a3-467b-401b-8d06-01d5c23898c8:9/24/2026 7:40:56 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T19:43:14.864Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T19:43:15.403Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T19:43:16.591Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=fcea6189c0137cabd5a1d08abcc1efccde7de99043ab10d6e0631ef6e7c8d654 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j7926656%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ffcea6189c0137cabd5a1d08abcc1efccde7de99043ab10d6e0631ef6e7c8d654\ntime=2026-09-24T19:50:36.313Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192892754366464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7135,"pr_state":"closed","pr_sha":"78d2e7cdedac95816c4452d91dfcae35ac93f66a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T18:40:35Z","started_at":"2026-09-24T18:40:35Z","completed_at":"2026-09-24T21:10:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T21:19:12.012189949Z","timing_metadata_checked_at":"2026-09-24T21:19:12.012189949Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T21:19:12.01495304Z","region_metadata_checked_at":"2026-09-24T21:19:12.01495304Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"dd738ad2ed71bd159afcf4387fa814709f8c34a764e0dba1fa00c15afdf2998b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192892754366464","occurred_at":"2026-09-24T18:40:35Z","failed_at":"alert","signature_id":"c23483f9fa6966a89f6cf4d0f9d5416c765a770957ef669010321a77abc1e041","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T20:13:00Z\n Ended: 2026-09-24T20:19:03Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4366464-mgmt-2\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-6c9ff9bdfc-65hlc\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-6c9ff9bdfc-65hlc (namespace kube-applier) on cluster ci01-j4366464-mgmt-2 has averaged 122.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"e333d3b729bb892b9715feb411d10c581b1ffdd9346b31d41f0114b762d02510","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192892754366464","occurred_at":"2026-09-24T18:40:35Z","failed_at":"alert","signature_id":"8cd39d3a27a9983e4c03864ddd77a3a279fd3ba43a01731a97e5915930fd1c60","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T20:06:17Z\n Ended: 2026-09-24T20:15:17Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4366464-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4366464-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"f961ac5e37190ec994e19618b81416bca8d573a815f0f49e865ba66eb15f8fe8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192892754366464","occurred_at":"2026-09-24T18:40:35Z","failed_at":"e2e","signature_id":"bf45d3a1f9dc91a9af5a7119dd1b60337e9f49919a6a81afba2cb9b9a6885627","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:154]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002fcb410\u003e: \n failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-jgw9qgklj7f5\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-jgw9qgklj7f5, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:154]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002fcb410\u003e: \n failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-jgw9qgklj7f5\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-jgw9qgklj7f5, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-jgw9qgklj7f5\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-jgw9qgklj7f5, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192859166380032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7045,"pr_state":"closed","pr_sha":"711d134f86fa2910df2819fd4922f71db33ae23c","final_merged_sha":"711d134f86fa2910df2819fd4922f71db33ae23c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T18:40:27Z","started_at":"2026-09-24T18:40:27Z","completed_at":"2026-09-24T19:59:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T20:06:12.744264551Z","timing_metadata_checked_at":"2026-09-24T20:06:12.744264551Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T20:06:12.746910339Z","region_metadata_checked_at":"2026-09-24T20:06:12.746910339Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"b33c4029dbce3a7737b813b000bace40cd0502212806f9c15aba0798ab178ae6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192859166380032","occurred_at":"2026-09-24T18:40:27Z","failed_at":"provision","signature_id":"3d35abb788cea68e7144d2ec2e84c98ce2744861ebc8486e8349a5f051f0df62","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-24T19:21:37.035Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-24T19:51:36.670Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-24T19:21:37.041921702Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6380032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6380032-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\"}\\n{\\\"time\\\":\\\"2026-09-24T19:21:38.603615087Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6380032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6380032-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-24T19:21:39.051190133Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6380032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6380032-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dbba05f95aaac62366d47b6f23fc0ba2f14aa49eae9bd82309cadecd8332be34","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j6380032-mgmt-2\",\"clusterName\":\"ci01-j638003","search_query":"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-24T19:21:37.041921702Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6380032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6380032-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\"}\\n{\\\"time\\\":\\\"2026-09-24T19:21:38.603615087Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6380032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6380032-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-24T19:21:39.051190133Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6380032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6380032-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed"}},{"row_id":"c290c9795a2ecb36964f267621c333c47872ecfa1926ea6e24af1d7f782a70ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192859166380032","occurred_at":"2026-09-24T18:40:27Z","failed_at":"alert","signature_id":"c8d2c94c46ea41bb6616321fc23a840d9b2ec094985319abefcb6af50ae87597","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T19:45:14Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j6380032-mgmt-1\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","dbba05f95aaac62366d47b6f23fc0ba2f14aa49eae9bd82309cadecd8332be34"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192838828199936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7068,"pr_state":"closed","pr_sha":"5d483559deea65ed903f146fc4f20cbbc522559d","final_merged_sha":"269e5d56bcfac6dc9f8d3ca87b7fb1ea84bf53b2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T18:40:22Z","started_at":"2026-09-24T18:40:22Z","completed_at":"2026-09-24T21:16:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T21:19:11.792644773Z","timing_metadata_checked_at":"2026-09-24T21:19:11.792644773Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T21:19:11.796165061Z","region_metadata_checked_at":"2026-09-24T21:19:11.796165061Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"70bda82ec25197393ac47737986afcacccf12f183fc386cdacf4a3de6808e738","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192838828199936","occurred_at":"2026-09-24T18:40:22Z","failed_at":"alert","signature_id":"3ecef63f876f1b80222fe16ba8dddbc997352451ab492a895459bf901b651a77","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T19:49:47Z\n Ended: 2026-09-24T20:24:50Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8199936-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8199936-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"b1b29a6634010bc86a917cc6f7bf70181e232b923afe7e29d1c0a90541f1855a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192838828199936","occurred_at":"2026-09-24T18:40:22Z","failed_at":"alert","signature_id":"78083eff97a88f0d621458e0dfb940e0f79aaa0a1700d409ae56663484ef417b","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T20:16:12Z\n Ended: 2026-09-24T20:31:59Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8199936-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-5d749b55f6-8gwhn\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-5d749b55f6-8gwhn (namespace kube-applier) on cluster ci01-j8199936-mgmt-1 has averaged 154.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-24T20:19:58Z\n Ended: 2026-09-24T20:27:03Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8199936-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-5c84d8774-f5pj4\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-5c84d8774-f5pj4 (namespace mgmt-agent) on cluster ci01-j8199936-mgmt-1 has averaged 125.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192810386624512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7009,"pr_state":"closed","pr_sha":"30ab7aab2c6cd7fcf1f6520d0a43a948520ff9af","final_merged_sha":"c83bdabaf08bdb32938b4526f3e12a94c26f811c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T18:40:16Z","started_at":"2026-09-24T18:40:16Z","completed_at":"2026-09-24T20:57:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T21:08:42.667934644Z","timing_metadata_checked_at":"2026-09-24T21:08:42.667934644Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T21:08:42.671787031Z","region_metadata_checked_at":"2026-09-24T21:08:42.671787031Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"2be9475f84303edae7b6b632e8c00ed10b9882fee58471da981c1260daf21bab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192810386624512","occurred_at":"2026-09-24T18:40:16Z","failed_at":"e2e","signature_id":"4de39a43d9c45741b51ae3bace1a381fb4260155eb8ea95a416b6fbf37a19a98","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc003143460\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dtj57jh59wlh\" to finish creating: GET https://rp.j6624512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7eb325fa-7f6c-4e10-8c9b-3bebf9248544\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7eb325fa-7f6c-4e10-8c9b-3bebf9248544\",\n \"name\": \"7eb325fa-7f6c-4e10-8c9b-3bebf9248544\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T19:46:17.078177969Z\",\n \"endTime\": \"2026-09-24T20:05:23.632105826Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l6n2u5f4n0l6i5m-np-delete-main-lqq79-j492z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dtj57jh59wlh--managed/providers/Microsoft.Compute/virtualMachines/l6n2u5f4n0l6i5m-np-delete-main-lqq79-j492z to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l6n2u5f4n0l6i5m-np-delete-main-lqq79-j492z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dtj57jh59wlh--managed/providers/Microsoft.Compute/virtualMachines/l6n2u5f4n0l6i5m-np-delete-main-lqq79-j492z to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc003143460\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dtj57jh59wlh\" to finish creating: GET https://rp.j6624512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7eb325fa-7f6c-4e10-8c9b-3bebf9248544\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7eb325fa-7f6c-4e10-8c9b-3bebf9248544\",\n \"name\": \"7eb325fa-7f6c-4e10-8c9b-3bebf9248544\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T19:46:17.078177969Z\",\n \"endTime\": \"2026-09-24T20:05:23.632105826Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l6n2u5f4n0l6i5m-np-delete-main-lqq79-j492z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dtj57jh59wlh--managed/providers/Microsoft.Compute/virtualMachines/l6n2u5f4n0l6i5m-np-delete-main-lqq79-j492z to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l6n2u5f4n0l6i5m-np-delete-main-lqq79-j492z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dtj57jh59wlh--managed/providers/Microsoft.Compute/virtualMachines/l6n2u5f4n0l6i5m-np-delete-main-lqq79-j492z to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-2tslxd58w56h\" to finish creating: GET https://rp"}},{"row_id":"9c97905478789328e8cceba66531d66b96f8e99f36fe5855d7e55c06ccc4f446","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192810386624512","occurred_at":"2026-09-24T18:40:16Z","failed_at":"e2e","signature_id":"b935a8a8e1ec9f7c77dd3ec089fb77afdc37f7a2e866b79eb951cee11d3e6a47","test_name":"KSM HCP Metrics metrics should be present in HCP Azure Monitoring Workspace","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:185]: Timed out after 1500.000s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:183 with:\nexpected kube_node_status_condition from kube-state-metrics; kube_node_info from kube-state-metrics but got no results\nExpected\n \u003c[]string | len:2, cap:2\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:185]: Timed out after 1500.000s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:183 with:\nexpected kube_node_status_condition from kube-state-metrics; kube_node_info from kube-state-metrics but got no results\nExpected\n \u003c[]string | len:2, cap:2\u003e: ...\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d91796eeda5752e6e2b49b398ec435f90a4ab6d3394ea2d0cd4ce5b2c2f30a11","failure_pattern":"not all KSM metrics appeared in Azure Monitor","search_query":"not all KSM metrics appeared in Azure Monitor"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","d91796eeda5752e6e2b49b398ec435f90a4ab6d3394ea2d0cd4ce5b2c2f30a11"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103191956178866176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7141,"pr_state":"closed","pr_sha":"ea57d380a37ac81b2267d05b54b0243732ffa7a6","final_merged_sha":"6afba8239efe6245441f25e86ccd2ab7ba7f5035","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T18:36:45Z","started_at":"2026-09-24T18:36:45Z","completed_at":"2026-09-24T21:02:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T21:08:43.049117452Z","timing_metadata_checked_at":"2026-09-24T21:08:43.049117452Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T21:08:43.05268904Z","region_metadata_checked_at":"2026-09-24T21:08:43.05268904Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"1ff4d5a717d8734b74dadb53c29b4109339bd548928a1d1bc2fb3fd478dde046","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103191956178866176","occurred_at":"2026-09-24T18:36:45Z","failed_at":"alert","signature_id":"7fbc78ba550f54a61f9bb8680a644c7d2c72d2a80281632662dd5cddec3c05d7","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 12 time(s) (1 unknown, 11 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T19:41:45Z\n Ended: 2026-09-24T20:10:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8866176-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t1io3vu7p3gua60c6258horequmjsus-f3a9j8b5h7f1o0j\", pod=\"router-5cc57c6448-t4p5n\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t1io3vu7p3gua60c6258horequmjsus-f3a9j8b5h7f1o0j/router-5cc57c6448-t4p5n has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"20ded35cb382ab307cd631080c54780b7a7535830af57a26a58a56a6b5e09fea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103191956178866176","occurred_at":"2026-09-24T18:36:45Z","failed_at":"e2e","signature_id":"43e7b49a53b9f05fe2662927a58e42b643a0bf4af76dedddd2bff94bc094d954","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d1d080\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-dkvsbz\" in resourcegroup=\"rg-np-version-upgrade-dkvsbz-sql98dwv9hds\" to finish creating: GET https://rp.j8866176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aa5258ae-6ac0-4393-ba90-fd7fff99d1fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aa5258ae-6ac0-4393-ba90-fd7fff99d1fd\",\n \"name\": \"aa5258ae-6ac0-4393-ba90-fd7fff99d1fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T19:41:58.103375269Z\",\n \"endTime\": \"2026-09-24T20:01:00.35851472Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c8k0y6n8u7a0y5r-npupgrade-4-20-ctsc2-95r6h: Failed: interfaces failed to create or update. err: failed to get existing resource rg-np-version-upgrade-dkvsbz-sql98dwv9hds-np-upgrade-dk-d348d3ab/c8k0y6n8u7a0y5r-npupgrade-4-20-ctsc2-95r6h-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-dkvsbz-sql98dwv9hds-np-upgrade-dk-d348d3ab/providers/Microsoft.Network/networkInterfaces/c8k0y6n8u7a0y5r-npupgrade-4-20-ctsc2-95r6h-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'a76148d4-cb11-4b93-9363-54ba8239b0b1' exceede... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d1d080\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-dkvsbz\" in resourcegroup=\"rg-np-version-upgrade-dkvsbz-sql98dwv9hds\" to finish creating: GET https://rp.j8866176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aa5258ae-6ac0-4393-ba90-fd7fff99d1fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aa5258ae-6ac0-4393-ba90-fd7fff99d1fd\",\n \"name\": \"aa5258ae-6ac0-4393-ba90-fd7fff99d1fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T19:41:58.103375269Z\",\n \"endTime\": \"2026-09-24T20:01:00.35851472Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c8k0y6n8u7a0y5r-npupgrade-4-20-ctsc2-95r6h: Failed: interfaces failed to create or update. err: failed to get existing resource rg-np-version-upgrade-dkvsbz-sql98dwv9hds-np-upgrade-dk-d348d3ab/c8k0y6n8u7a0y5r-npupgrade-4-20-ctsc2-95r6h-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-dkvsbz-sql98dwv9hds-np-upgrade-dk-d348d3ab/providers/Microsoft.Network/networkInterfaces/c8k0y6n8u7a0y5r-npupgrade-4-20-ctsc2-95r6h-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'a76148d4-cb11-4b93-9363-54ba8239b0b1' exceede... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c62a41b1ffa48cd7eb5519e7f4bdc548f260009543ddaf09220fa0719901910a","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceede... message truncated; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"21a3bc4d2352787fcbfdee318ff74b93f1dfd01d31fb619bed16a6784219c569","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103191956178866176","occurred_at":"2026-09-24T18:36:45Z","failed_at":"e2e","signature_id":"c29dc181d38e6ceb97183fd208302b380dff64e5d9a1ab9ca671c1309bb44d49","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-c7xkx7wln57r/cluster-candidate-5-0-87wr9p should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010cc280\u003e: \n failed to create HCP cluster cluster-candidate-5-0-87wr9p: failed waiting for cluster=\"cluster-candidate-5-0-87wr9p\" in resourcegroup=\"rg-candidate-5-0-c7xkx7wln57r\" to finish creating: GET https://rp.j8866176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d9d1e9a3-4f45-45e2-a7a8-d0765c81f904\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d9d1e9a3-4f45-45e2-a7a8-d0765c81f904\",\n \"name\": \"d9d1e9a3-4f45-45e2-a7a8-d0765c81f904\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T19:29:18.072972108Z\",\n \"endTime\": \"2026-09-24T19:58:19.977586305Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.3 is Partial (want Completed), started 22m42s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-c7xkx7wln57r/cluster-candidate-5-0-87wr9p should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010cc280\u003e: \n failed to create HCP cluster cluster-candidate-5-0-87wr9p: failed waiting for cluster=\"cluster-candidate-5-0-87wr9p\" in resourcegroup=\"rg-candidate-5-0-c7xkx7wln57r\" to finish creating: GET https://rp.j8866176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d9d1e9a3-4f45-45e2-a7a8-d0765c81f904\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d9d1e9a3-4f45-45e2-a7a8-d0765c81f904\",\n \"name\": \"d9d1e9a3-4f45-45e2-a7a8-d0765c81f904\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T19:29:18.072972108Z\",\n \"endTime\": \"2026-09-24T19:58:19.977586305Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.3 is Partial (want Completed), started 22m42s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","c62a41b1ffa48cd7eb5519e7f4bdc548f260009543ddaf09220fa0719901910a"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6866/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103189605326000128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6866,"pr_state":"open","pr_sha":"0992d159fbfe175e8b331609e6bb213f2338c445","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T18:27:31Z","started_at":"2026-09-24T18:27:31Z","completed_at":"2026-09-24T18:36:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T18:41:54.93502148Z","timing_metadata_checked_at":"2026-09-24T18:41:54.93502148Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-24T18:41:54.93780507Z","region_metadata_checked_at":"2026-09-24T19:45:13.381455799Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"5e45f892d0da413b93fe469fc6f695c5c20f5ae4b907e24d5add095482b462f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6866/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103189605326000128","occurred_at":"2026-09-24T18:27:31Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103186015765139456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6920,"pr_state":"open","pr_sha":"0ed9e5d46a7c4f7ca3e7d722b7dc05b5b2dacb37","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T18:13:16Z","started_at":"2026-09-24T18:13:16Z","completed_at":"2026-09-24T19:05:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T19:13:41.97764333Z","timing_metadata_checked_at":"2026-09-24T19:13:41.97764333Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T19:13:41.981034818Z","region_metadata_checked_at":"2026-09-24T19:13:41.981034818Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"075e2b6bf5eae27014c980405631eb76a2dc585758b93dc53a003d4bda3086a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103186015765139456","occurred_at":"2026-09-24T18:13:16Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-24T18:29:09.876Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T18:29:11.341Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T18:29:12.388Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=46262dae9ca2d4432bcb86d812efc357631d2858f6b01cf69a97fe136d2ea262 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5139456%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F46262dae9ca2d4432bcb86d812efc357631d2858f6b01cf69a97fe136d2ea262\ntime=2026-09-24T18:31:42.746Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5139456/providers/Microsoft.Resources/deployments/46262dae9ca2d4432bcb86d812efc357631d2858f6b01cf69a97fe136d2ea262/operationStatuses/08584113323338045976\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5139456/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5139456/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j5139456\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report b5b7aacb-f0b5-4b54-adeb-991d4fa8294a:9/24/2026 6:30:21 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T18:33:42.752Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T18:33:43.572Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T18:33:44.582Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=45a3917d216f8340d66b25a24675844c0acbd8aeeae2f0b40c3b1500b09fb2b7 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5139456%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F45a3917d216f8340d66b25a24675844c0acbd8aeeae2f0b40c3b1500b09fb2b7\ntime=2026-09-24T18:49:16.840Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5139456/providers/Microsoft.Resources/deployments/45a3917d216f8340d66b25a24675844c0acbd8aeeae2f0b40c3b1500b09fb2b7/operationStatuses/08584113320616539397\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5139456/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5139456/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j5139456\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 3eef3d2c-b471-47fb-94a5-37504c4f3120:9/24/2026 6:48:59 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T18:51:16.840Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T18:51:17.503Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T18:51:18.086Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=15f82a2bd039b9788fc4c6a28262d04cab0bbaa6e1b0a641e2cdc7833b13a47d portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5139456%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F15f82a2bd039b9788fc4c6a28262d04cab0bbaa6e1b0a641e2cdc7833b13a47d\ntime=2026-09-24T18:59:09.655Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103182792887111680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6993,"pr_state":"closed","pr_sha":"54fa83b6be2bda741a3e467491a8b8caf8a2dbe3","final_merged_sha":"1a558ec553718fc2df21030c08516ca31075af74","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T18:00:08Z","started_at":"2026-09-24T18:00:08Z","completed_at":"2026-09-24T20:10:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T20:16:23.013851257Z","timing_metadata_checked_at":"2026-09-24T20:16:23.013851257Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T20:16:23.016629047Z","region_metadata_checked_at":"2026-09-24T20:16:23.016629047Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2c6125c6149e9f168544e52097c171aacd8b38f4258096a153f45e974295f8ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103182792887111680","occurred_at":"2026-09-24T18:00:08Z","failed_at":"e2e","signature_id":"5bf349cb8ec9f251b4fddcd8eba4f881e8bb266c023ecc7561970e4ac0e37274","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc002703dd0\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-x45qskfpvk4k\" to finish creating: GET https://rp.j7111680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50f10e36-adf3-41e9-a281-5a223249a720\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50f10e36-adf3-41e9-a281-5a223249a720\",\n \"name\": \"50f10e36-adf3-41e9-a281-5a223249a720\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T18:57:03.302695229Z\",\n \"endTime\": \"2026-09-24T19:16:10.982967273Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-djkxv-44p2x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-x45qskfpvk4k--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-djkxv-44p2x to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-djkxv-44p2x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-x45qskfpvk4k--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-djkxv-44p2x to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc002703dd0\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-x45qskfpvk4k\" to finish creating: GET https://rp.j7111680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50f10e36-adf3-41e9-a281-5a223249a720\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50f10e36-adf3-41e9-a281-5a223249a720\",\n \"name\": \"50f10e36-adf3-41e9-a281-5a223249a720\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T18:57:03.302695229Z\",\n \"endTime\": \"2026-09-24T19:16:10.982967273Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-djkxv-44p2x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-x45qskfpvk4k--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-djkxv-44p2x to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-djkxv-44p2x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-x45qskfpvk4k--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-djkxv-44p2x to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"faaf4b1f5915530cc590a6d965e51a07e2893bcc928bcf257ec4329e7f00397f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-djkxv-\u003cid\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-djkxv-\u003cid\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["faaf4b1f5915530cc590a6d965e51a07e2893bcc928bcf257ec4329e7f00397f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103181727441620992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6915,"pr_state":"open","pr_sha":"e207b76e733cb65c677c16c7f38706c45baff4cd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T17:56:13Z","started_at":"2026-09-24T17:56:13Z","completed_at":"2026-09-24T18:54:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T19:02:59.929360661Z","timing_metadata_checked_at":"2026-09-24T19:02:59.929360661Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T19:02:59.93226675Z","region_metadata_checked_at":"2026-09-24T19:02:59.93226675Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a8b21965e23b40697ad31377bbff85aeff1193c959094df27b5187c21c003536","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103181727441620992","occurred_at":"2026-09-24T17:56:13Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-24T18:17:20.985Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T18:17:22.770Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T18:17:23.796Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=7b4c34179719ab5cc2ae13b15dfea6825041e69bc908ddbe35da23b6e91532fe portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1620992%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F7b4c34179719ab5cc2ae13b15dfea6825041e69bc908ddbe35da23b6e91532fe\ntime=2026-09-24T18:19:54.241Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1620992/providers/Microsoft.Resources/deployments/7b4c34179719ab5cc2ae13b15dfea6825041e69bc908ddbe35da23b6e91532fe/operationStatuses/08584113330423705230\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1620992/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1620992/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1620992\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 4a57e20a-1d76-48b2-b50a-28e5782b1b82:9/24/2026 6:18:40 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T18:21:54.243Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T18:21:54.963Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T18:21:55.659Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=2924362328972c1641489d92ac02150d2b594d769a16699f5d262112aa0bcea2 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1620992%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F2924362328972c1641489d92ac02150d2b594d769a16699f5d262112aa0bcea2\ntime=2026-09-24T18:37:27.781Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1620992/providers/Microsoft.Resources/deployments/2924362328972c1641489d92ac02150d2b594d769a16699f5d262112aa0bcea2/operationStatuses/08584113327702852521\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1620992/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1620992/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1620992\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 9d5875b5-3dc1-48d4-ad37-aeece309c94b:9/24/2026 6:37:03 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T18:39:27.781Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T18:39:28.357Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T18:39:28.967Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=3fd966b63ec6f54103e9b3db5ef1b5f3378370e79e818f3e994ee1c8afb25dfc portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1620992%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F3fd966b63ec6f54103e9b3db5ef1b5f3378370e79e818f3e994ee1c8afb25dfc\ntime=2026-09-24T18:47:20.585Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7139/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103179973853450240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7139,"pr_state":"open","pr_sha":"dc6fdc7c387610a5089bc44bba9a53c4d5715ff0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-24T17:49:15Z","started_at":"2026-09-24T17:49:15Z","completed_at":"2026-09-24T20:00:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T20:06:14.286477998Z","timing_metadata_checked_at":"2026-09-24T20:06:14.286477998Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T20:06:14.289007686Z","region_metadata_checked_at":"2026-09-24T20:06:14.289007686Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103178818297532416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7105,"pr_state":"open","pr_sha":"8cb3a165b86731107795b1de4deb44a21f84903c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-24T17:44:40Z","started_at":"2026-09-24T17:44:40Z","completed_at":"2026-09-24T20:02:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T20:06:14.144722437Z","timing_metadata_checked_at":"2026-09-24T20:06:14.144722437Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T20:06:14.147936822Z","region_metadata_checked_at":"2026-09-24T20:06:14.147936822Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103178709123993600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7054,"pr_state":"closed","pr_sha":"f0e1fa3e33ffc30bc4eeec511d85f73874d58b9b","final_merged_sha":"7553575593411fb5f3c1d57088aa752a234c76d1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T17:44:13Z","started_at":"2026-09-24T17:44:13Z","completed_at":"2026-09-24T18:47:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T18:52:12.749391069Z","timing_metadata_checked_at":"2026-09-24T18:52:12.749391069Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T18:52:12.752159059Z","region_metadata_checked_at":"2026-09-24T18:52:12.752159059Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"7b078c5af9be994edc291e28a6e019e3ae73e67c41438a4f3f329cd7cc48c7d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103178709123993600","occurred_at":"2026-09-24T17:44:13Z","failed_at":"provision","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"templatize-pipeline","failure_text":"time=2026-09-24T18:10:29.558Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T18:10:31.174Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T18:10:32.148Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=a8e2c4fffaed9a83683c76f4cd1103013606b70083bc71a47e08d5db377d9214 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3993600%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fa8e2c4fffaed9a83683c76f4cd1103013606b70083bc71a47e08d5db377d9214\ntime=2026-09-24T18:13:03.874Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3993600/providers/Microsoft.Resources/deployments/a8e2c4fffaed9a83683c76f4cd1103013606b70083bc71a47e08d5db377d9214/operationStatuses/08584113334539025042\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3993600/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3993600/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j3993600\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 22bdd29b-5870-48fa-a5e1-6c0d59fd6876:9/24/2026 6:11:56 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T18:15:03.880Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T18:15:04.725Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T18:15:05.364Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=8a4a60a246ff614765ccadb769a101a106359c09306b43be6c41f12415096ee7 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3993600%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F8a4a60a246ff614765ccadb769a101a106359c09306b43be6c41f12415096ee7\ntime=2026-09-24T18:30:38.634Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3993600/providers/Microsoft.Resources/deployments/8a4a60a246ff614765ccadb769a101a106359c09306b43be6c41f12415096ee7/operationStatuses/08584113331805138432\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3993600/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3993600/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j3993600\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 9017f3bb-de18-4e0c-bc08-e4ca6d73d0b2:9/24/2026 6:30:11 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T18:32:38.634Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T18:32:39.201Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T18:32:40.145Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=32815c49484338bcfc36246761db2e6b35d8eeae2d30dde766c8906cf26fc1ee portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3993600%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F32815c49484338bcfc36246761db2e6b35d8eeae2d30dde766c8906cf26fc1ee\ntime=2026-09-24T18:40:29.143Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103175794774773760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7053,"pr_state":"closed","pr_sha":"b938b99cc7c5a09bd230dfa6d470fb9e6a8d6e0a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T17:32:39Z","started_at":"2026-09-24T17:32:39Z","completed_at":"2026-09-24T19:55:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T20:06:14.038358917Z","timing_metadata_checked_at":"2026-09-24T20:06:14.038358917Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T20:06:14.041531502Z","region_metadata_checked_at":"2026-09-24T20:06:14.041531502Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"69190e498b4a7db8b091adba46d9fec8ec6817b180026b704d4cc4e700cb269a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103175794774773760","occurred_at":"2026-09-24T17:32:39Z","failed_at":"alert","signature_id":"e7b9c2ed622a10fb2e558d1618704e5e291864f9724b4b5ab5eb019d5bcadf46","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T18:55:22Z\n Ended: 2026-09-24T19:10:23Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4773760-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-657fcdbc5f-zj5cq\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-657fcdbc5f-zj5cq (namespace kube-applier) on cluster ci01-j4773760-mgmt-1 has averaged 138.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103170613366493184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7117,"pr_state":"closed","pr_sha":"ea43f7ba6df9f089661d7c30a490eb1e026104e9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T17:12:03Z","started_at":"2026-09-24T17:12:03Z","completed_at":"2026-09-24T19:24:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T19:34:20.583247608Z","timing_metadata_checked_at":"2026-09-24T19:34:20.583247608Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T19:34:20.585838199Z","region_metadata_checked_at":"2026-09-24T19:34:20.585838199Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"a9c0ae27d91a19db34dd6f578ed1e1df0bc999dca929e1f78d77b540d0e82765","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103170613366493184","occurred_at":"2026-09-24T17:12:03Z","failed_at":"alert","signature_id":"f850918927d721d1a9c993363a6f66eb1a0457832ced1c9c9d5463e320ffdb40","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T18:02:26Z\n Ended: 2026-09-24T18:36:26Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6493184-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6493184-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6967/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103169632058740736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6967,"pr_state":"open","pr_sha":"cee12ce771cd652c2b31b8de85b68f0b1b2ea9cd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-24T17:08:09Z","started_at":"2026-09-24T17:08:09Z","completed_at":"2026-09-24T19:21:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T19:34:20.449273455Z","timing_metadata_checked_at":"2026-09-24T19:34:20.449273455Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T19:34:20.452866143Z","region_metadata_checked_at":"2026-09-24T19:34:20.452866143Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7138/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103169546784346112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7138,"pr_state":"closed","pr_sha":"419da749571f03657717e4b28dfecec9ed7be51d","final_merged_sha":"419da749571f03657717e4b28dfecec9ed7be51d","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-24T17:07:49Z","started_at":"2026-09-24T17:07:49Z","completed_at":"2026-09-24T19:21:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T19:34:20.701636512Z","timing_metadata_checked_at":"2026-09-24T19:34:20.701636512Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T19:34:20.704603902Z","region_metadata_checked_at":"2026-09-24T19:34:20.704603902Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7137/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103164909851250688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7137,"pr_state":"closed","pr_sha":"44d9699680d2624d642230ae9206e30fd311ddb0","final_merged_sha":"44d9699680d2624d642230ae9206e30fd311ddb0","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-24T16:49:22Z","started_at":"2026-09-24T16:49:22Z","completed_at":"2026-09-24T19:02:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T19:13:42.592365936Z","timing_metadata_checked_at":"2026-09-24T19:13:42.592365936Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T19:13:42.595300726Z","region_metadata_checked_at":"2026-09-24T19:13:42.595300726Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7090/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103164576785764352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7090,"pr_state":"closed","pr_sha":"6747bd2682b02d1bbe260d5e69aab3a33431b9ad","final_merged_sha":"6747bd2682b02d1bbe260d5e69aab3a33431b9ad","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-24T16:48:04Z","started_at":"2026-09-24T16:48:04Z","completed_at":"2026-09-24T19:11:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T19:24:10.014047672Z","timing_metadata_checked_at":"2026-09-24T19:24:10.014047672Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T19:24:10.017860956Z","region_metadata_checked_at":"2026-09-24T19:24:10.017860956Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103162932127862784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6821,"pr_state":"closed","pr_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","final_merged_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T16:41:32Z","started_at":"2026-09-24T16:41:32Z","completed_at":"2026-09-24T18:49:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T18:52:12.421617687Z","timing_metadata_checked_at":"2026-09-24T18:52:12.421617687Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T18:52:12.424175378Z","region_metadata_checked_at":"2026-09-24T18:52:12.424175378Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b73168907384be58fa39cb3f49df0cb6693b64c0050322e016d5e69bce7ef787","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103162932127862784","occurred_at":"2026-09-24T16:41:32Z","failed_at":"e2e","signature_id":"71ad104d7c45702c68a9e44ecbe6eb8ff08e7116726d4744adcdde9ab6895dc5","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cea980\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-dx85jrjs9gbf\" to finish creating: GET https://rp.j7862784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ddf0b59-a1c5-4f88-9e2a-09c2b8b441c9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ddf0b59-a1c5-4f88-9e2a-09c2b8b441c9\",\n \"name\": \"9ddf0b59-a1c5-4f88-9e2a-09c2b8b441c9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T17:42:43.803875728Z\",\n \"endTime\": \"2026-09-24T18:01:44.201124036Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j3g0e5m6b3n7n1m-np-1-45jtt-ld7k7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-dx85jrjs9gbf--managed/providers/Microsoft.Compute/virtualMachines/j3g0e5m6b3n7n1m-np-1-45jtt-ld7k7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j3g0e5m6b3n7n1m-np-1-45jtt-ld7k7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-dx85jrjs9gbf--managed/providers/Microsoft.Compute/virtualMachines/j3g0e5m6b3n7n1m-np-1-45jtt-ld7k7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cea980\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-dx85jrjs9gbf\" to finish creating: GET https://rp.j7862784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ddf0b59-a1c5-4f88-9e2a-09c2b8b441c9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ddf0b59-a1c5-4f88-9e2a-09c2b8b441c9\",\n \"name\": \"9ddf0b59-a1c5-4f88-9e2a-09c2b8b441c9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T17:42:43.803875728Z\",\n \"endTime\": \"2026-09-24T18:01:44.201124036Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j3g0e5m6b3n7n1m-np-1-45jtt-ld7k7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-dx85jrjs9gbf--managed/providers/Microsoft.Compute/virtualMachines/j3g0e5m6b3n7n1m-np-1-45jtt-ld7k7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j3g0e5m6b3n7n1m-np-1-45jtt-ld7k7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-dx85jrjs9gbf--managed/providers/Microsoft.Compute/virtualMachines/j3g0e5m6b3n7n1m-np-1-45jtt-ld7k7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"eacd13fa6af2b06f56ce1295197593af50aee51808239f944fd6ca6273414a50","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["eacd13fa6af2b06f56ce1295197593af50aee51808239f944fd6ca6273414a50"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7130/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103162925303730176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7130,"pr_state":"closed","pr_sha":"8b117a5321fa82d95903dd74ddf66560af88c52e","final_merged_sha":"8b117a5321fa82d95903dd74ddf66560af88c52e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-24T16:41:30Z","started_at":"2026-09-24T16:41:30Z","completed_at":"2026-09-24T19:03:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T19:13:42.371888988Z","timing_metadata_checked_at":"2026-09-24T19:13:42.371888988Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T19:13:42.374811478Z","region_metadata_checked_at":"2026-09-24T19:13:42.374811478Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103162898384687104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7009,"pr_state":"closed","pr_sha":"30ab7aab2c6cd7fcf1f6520d0a43a948520ff9af","final_merged_sha":"c83bdabaf08bdb32938b4526f3e12a94c26f811c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T16:41:24Z","started_at":"2026-09-24T16:41:24Z","completed_at":"2026-09-24T17:32:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T17:37:56.759889752Z","timing_metadata_checked_at":"2026-09-24T17:37:56.759889752Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T17:37:56.763089641Z","region_metadata_checked_at":"2026-09-24T17:37:56.763089641Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c0da1e76ffc42932c2ae173f0af4426712bee2ada4cfaccb17695d0de47be790","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103162898384687104","occurred_at":"2026-09-24T16:41:24Z","failed_at":"provision","signature_id":"d465f96a6ebaf98458f369563694ed7b4f0235597cd9af66af75425f966c9780","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-24T17:25:13.342Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 description=\"Step nsp\\n Kind: ARM\\n Template: templates/mgmt-nsp.bicep\\n Parameters: configurations/mgmt-nsp.tmpl.bicepparam\"\ntime=2026-09-24T17:25:13.641Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2\ntime=2026-09-24T17:25:14.167Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 deployment=9d3f2ed9b958df750e3b7f101250ebe891c1a35bc943187871975b34cdf31ee3 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4687104-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9d3f2ed9b958df750e3b7f101250ebe891c1a35bc943187871975b34cdf31ee3\ntime=2026-09-24T17:25:44.283Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4687104-mgmt-2/providers/Microsoft.Resources/deployments/9d3f2ed9b958df750e3b7f101250ebe891c1a35bc943187871975b34cdf31ee3/operationStatuses/08584113361715956682\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4687104-mgmt-2/providers/Microsoft.Resources/deployments/profile-bmfsiknuehg4q\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4687104-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j4687104-mgmt-2/resourceAssociations/nsp-j4687104-mgmt-2-ra-d68332b2-0c36-56b4-8794-de05fd33fea0\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"HttpRequestException\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Error while processing request for association /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4687104-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j4687104-mgmt-2/resourceAssociations/nsp-j4687104-mgmt-2-ra-d68332b2-0c36-56b4-8794-de05fd33fea0, Error message: An error occurred while sending the request.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9220c54af3cdcf82280969ee7ca39e91829e564fcd5ac622742ec149542a29fd","failure_pattern":"ERROR CODE: DeploymentFailed; detail code HttpRequestException; detail message Error while processing network security perimeter association: An error occurred while sending the request.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9220c54af3cdcf82280969ee7ca39e91829e564fcd5ac622742ec149542a29fd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103161541376348160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7123,"pr_state":"closed","pr_sha":"c07c99eb845db32bdffea8c5c152b60fa095f319","final_merged_sha":"c07c99eb845db32bdffea8c5c152b60fa095f319","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T16:36:00Z","started_at":"2026-09-24T16:36:00Z","completed_at":"2026-09-24T19:03:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T19:13:42.156316222Z","timing_metadata_checked_at":"2026-09-24T19:13:42.156316222Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T19:13:42.15981281Z","region_metadata_checked_at":"2026-09-24T19:13:42.15981281Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"2e95dd71973747027c0e0f5422a54a558cde924346677d7433748787c6079e50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103161541376348160","occurred_at":"2026-09-24T16:36:00Z","failed_at":"alert","signature_id":"2c5be9a399728a0c355066d00beeef8ebab315261af0ce0681e717060c90e781","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T18:05:21Z\n Ended: 2026-09-24T18:14:22Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6348160-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6348160-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103161107324604416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6820,"pr_state":"open","pr_sha":"c99d95a578f89e3e4c3bfc476a031bb4b59056f4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T16:34:17Z","started_at":"2026-09-24T16:34:17Z","completed_at":"2026-09-24T17:54:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T17:59:20.649404865Z","timing_metadata_checked_at":"2026-09-24T17:59:20.649404865Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T17:59:20.652764553Z","region_metadata_checked_at":"2026-09-24T17:59:20.652764553Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"2916bf110b1f578da1fe53be31cc95ac21feeb39aa76f579f3e49034f684a7cd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103161107324604416","occurred_at":"2026-09-24T16:34:17Z","failed_at":"alert","signature_id":"7726a18972655e8be5469370e6d72d78ec561de245d068ff794d9ec202b33f3d","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T17:34:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j4604416-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.41:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4604416/providers/microsoft.monitor/accounts/services-j4604416\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-849cb69576-mmg7d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-24T17:34:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j4604416-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.41:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4604416/providers/microsoft.monitor/accounts/services-j4604416\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-849cb69576-mmg7d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"c18f40d241d9744ba53ec8146fe9f3c24b69db7da5ee7f643e04aa5b27fd5352","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103161107324604416","occurred_at":"2026-09-24T16:34:17Z","failed_at":"provision","signature_id":"09989fcafa95c4bca4519a71c6fa418fa3c6af4fbe8bd7b270ead898a41c8b5c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","test_suite":"templatize-pipeline","failure_text":"time=2026-09-24T17:17:41.012Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-24T17:17:41.580Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j4604416-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j4604416-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:7af6cb650e29c367219bebbce4020e6d53088d22e07ab5040d4b4a03685e7176 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:4dc69aba-ed69-4d27-a425-df6d14df249a region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#4dc69aba-ed69-4d27-a425-df6d14df249a INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-24T17:17:41.580Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:17:41.794Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-24T17:17:42.103Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-24T17:17:42.103Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:17:42.232Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:27:46.168Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-24T17:27:46.168Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-24T17:28:46.169Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-24T17:28:47.119Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j4604416-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j4604416-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:7af6cb650e29c367219bebbce4020e6d53088d22e07ab5040d4b4a03685e7176 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:4dc69aba-ed69-4d27-a425-df6d14df249a region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#4dc69aba-ed69-4d27-a425-df6d14df249a INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-24T17:28:47.119Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:28:47.488Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-24T17:28:48.000Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:28:48.000Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:28:48.185Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-24T17:28:49.205Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:28:49.216Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:28:49.680Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:28:50.092Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-24T17:28:50.620Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-24T17:28:50.999Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:28:51.506Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:28:51.949Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:28:52.233Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-24T17:28:52.233Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:28:52.429Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-24T17:38:55.132Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-24T17:38:55.133Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-24T17:39:55.134Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-24T17:39:56.017Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j4604416-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j4604416-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:7af6cb650e29c367219bebbce4020e6d53088d22e07ab5040d4b4a03685e7176 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:4dc69aba-ed69-4d27-a425-df6d14df249a region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#4dc69aba-ed69-4d27-a425-df6d14df249a INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-24T17:39:56.017Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:39:56.414Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-24T17:39:56.965Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:39:56.965Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:39:57.150Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-24T17:39:58.124Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:39:58.124Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:39:58.718Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:39:59.129Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-24T17:39:59.937Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-24T17:40:00.244Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:40:00.730Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:40:01.347Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:40:01.716Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-24T17:40:01.716Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:40:01.972Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-24T17:47:41.098Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-24T17:47:41.098Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","failure_pattern":"failed to roll out Helm release: context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded"}},{"row_id":"d949633ea172af87659cbe11c25f145f3f396be677153c29192f2d5f2fda81db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103161107324604416","occurred_at":"2026-09-24T16:34:17Z","failed_at":"alert","signature_id":"679aa0117967b7ad541fd1b96fee91e26fcf0b44074834ec0f9774a9ce5ea7a4","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T17:33:47Z\n Severity: Sev3\n Labels: alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j4604416-svc\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"warning\"\n Description: The Arobit forwarder metrics endpoint on cluster ci01-j4604416-svc has been unreachable for 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired"}},{"row_id":"e24cc9e9079c96bf617d0c6a7633751a2330d374d17554f087d378e06651f38a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103161107324604416","occurred_at":"2026-09-24T16:34:17Z","failed_at":"alert","signature_id":"39ce01c9a16cf6ef63990abb5b52786f61c0e8bb85c12b3c8bff45bc21ee3246","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T17:24:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4604416-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-fw4cp\", region=\"westus3\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-fw4cp has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-24T17:24:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4604416-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-dfd7c\", region=\"westus3\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-dfd7c has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-24T17:24:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4604416-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-vflzc\", region=\"westus3\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-vflzc has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-24T17:24:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4604416-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-fn4cr\", region=\"westus3\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-fn4cr has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103155003655983104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7068,"pr_state":"closed","pr_sha":"5d483559deea65ed903f146fc4f20cbbc522559d","final_merged_sha":"269e5d56bcfac6dc9f8d3ca87b7fb1ea84bf53b2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T16:09:57Z","started_at":"2026-09-24T16:09:57Z","completed_at":"2026-09-24T18:32:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T18:41:55.564976289Z","timing_metadata_checked_at":"2026-09-24T18:41:55.564976289Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T18:41:55.57339326Z","region_metadata_checked_at":"2026-09-24T18:41:55.57339326Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"e2ec098f7fadc115b24c7cb094109a37cd68cc1a6040c33934562a940a738b91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103155003655983104","occurred_at":"2026-09-24T16:09:57Z","failed_at":"alert","signature_id":"fe0efdc1006e609d9473efff132bf7955c32a6ea999c6a2e2fb575020cee0852","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T17:33:17Z\n Ended: 2026-09-24T17:43:15Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j5983104-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j5983104-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103154812324417536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6970,"pr_state":"closed","pr_sha":"a65742c649839e7391c082c1b1d3d7fd6049518c","final_merged_sha":"af55f114edf1b1d398480eeee6a41ae706e821e2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T16:09:16Z","started_at":"2026-09-24T16:09:16Z","completed_at":"2026-09-24T18:27:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T18:31:06.531085182Z","timing_metadata_checked_at":"2026-09-24T18:31:06.531085182Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T18:31:06.533814673Z","region_metadata_checked_at":"2026-09-24T18:31:06.533814673Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"0dcf2d2ef3e427d767c7035768455cf2f87e5641585b186eb81618f7c664c849","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103154812324417536","occurred_at":"2026-09-24T16:09:16Z","failed_at":"alert","signature_id":"1d82549097eef2d75be7580f57f3df55a47f5833cfb899cc75ff5cc5acc415fc","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T17:05:26Z\n Ended: 2026-09-24T17:39:24Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4417536-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4417536-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"410a53ee5f695ffd3e3460a9cb7acf41ae2d0e4e368f7e6522ca25de82965725","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103154812324417536","occurred_at":"2026-09-24T16:09:16Z","failed_at":"alert","signature_id":"66131251b5fecbeb3371e744344966e17e34b09c66d06812a5e0fe0811e83ed1","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T18:08:30Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j4417536-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-2fbf2\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-2fbf2 (namespace arobit) on cluster ci01-j4417536-svc has averaged 125.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103148398205997056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5562,"pr_state":"closed","pr_sha":"18cce3707ed7df30b1a12d9779a8035dbdc24710","final_merged_sha":"18cce3707ed7df30b1a12d9779a8035dbdc24710","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-24T15:43:45Z","started_at":"2026-09-24T15:43:45Z","completed_at":"2026-09-24T17:59:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T18:10:14.965462752Z","timing_metadata_checked_at":"2026-09-24T18:10:14.965462752Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T18:10:14.968703042Z","region_metadata_checked_at":"2026-09-24T18:10:14.968703042Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7110/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146547498717184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7110,"pr_state":"closed","pr_sha":"44357233f8c544a15f50f6aa86714aeae138b456","final_merged_sha":"ce66d7285ab6bfc8f735b0e7c24632e94158eb25","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-24T15:36:26Z","started_at":"2026-09-24T15:36:26Z","completed_at":"2026-09-24T18:03:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T18:10:16.248658982Z","timing_metadata_checked_at":"2026-09-24T18:10:16.248658982Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T18:10:16.251609772Z","region_metadata_checked_at":"2026-09-24T18:10:16.251609772Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146310981914624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7045,"pr_state":"closed","pr_sha":"711d134f86fa2910df2819fd4922f71db33ae23c","final_merged_sha":"711d134f86fa2910df2819fd4922f71db33ae23c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T15:35:16Z","started_at":"2026-09-24T15:35:16Z","completed_at":"2026-09-24T18:00:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T18:10:15.794590058Z","timing_metadata_checked_at":"2026-09-24T18:10:15.794590058Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T18:10:15.797886147Z","region_metadata_checked_at":"2026-09-24T18:10:15.797886147Z"},"failed_at":["alert"],"failed_test_count":4,"occurrences":[{"row_id":"3cb4687af9e28a63f2d75ae504f05dea78c1f9b70e2d113ff1087fa89d754d52","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146310981914624","occurred_at":"2026-09-24T15:35:16Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T16:50:03Z\n Ended: 2026-09-24T16:57:05Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1914624-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"8f64aa1ac303f32667e253588fa6a2815f08c10cbeb66352dbfbf2c291052339","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146310981914624","occurred_at":"2026-09-24T15:35:16Z","failed_at":"alert","signature_id":"4e2ec19ff9d069c5bb64045806ef329a6aca9db297b07736e4b201ead6847e07","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T17:41:02Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j1914624-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-8vzwj\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-8vzwj (namespace arobit) on cluster ci01-j1914624-svc has averaged 126.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"f5bc405d9d37aef65466170aea4bb576a9ec96a6634f6891ff38e2b13cba8a50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146310981914624","occurred_at":"2026-09-24T15:35:16Z","failed_at":"alert","signature_id":"0e8f57e37d73e9d51e134e1e0ceb298fcf33a07521e65f7fbe44784034a97653","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T17:31:32Z\n Ended: 2026-09-24T17:39:31Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j1914624-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j1914624-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"f6d3e3f02672ab7385d762b29bdb29d54c211b114491630c1101dc18f26696f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146310981914624","occurred_at":"2026-09-24T15:35:16Z","failed_at":"alert","signature_id":"342114364ff6caa404513f6899c73460c984413773999113333d0dc8ff64a1da","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T17:06:29Z\n Ended: 2026-09-24T17:15:28Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1914624-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1914624-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146302601695232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7117,"pr_state":"closed","pr_sha":"ea43f7ba6df9f089661d7c30a490eb1e026104e9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T15:35:15Z","started_at":"2026-09-24T15:35:15Z","completed_at":"2026-09-24T16:50:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T16:55:50.510664385Z","timing_metadata_checked_at":"2026-09-24T16:55:50.510664385Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T16:55:50.513918475Z","region_metadata_checked_at":"2026-09-24T16:55:50.513918475Z"},"failed_at":["alert","provision"],"failed_test_count":3,"occurrences":[{"row_id":"302dbb4f34436bd5ba2f436088b26f12e1b96239169c4119e920c16214c54782","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146302601695232","occurred_at":"2026-09-24T15:35:15Z","failed_at":"alert","signature_id":"49174f0842b090006dcda4660be00bccd297316410781deb14da7700d135010d","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T16:38:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1695232-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-40199610-vmss000000-j9kbh\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod kube-system/eraser-aks-system-40199610-vmss000000-j9kbh has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-24T16:38:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1695232-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-nffxs\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod kube-system/aks-secrets-store-csi-driver-nffxs has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-24T16:38:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1695232-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-2szd2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod kube-system/ama-metrics-node-2szd2 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-24T16:40:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1695232-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"swift-recorder\", pod=\"swift-recorder-tw7kk\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod swift-recorder/swift-recorder-tw7kk has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"76e36b3c8c8fcd36953ec47e438448774edde00517e6dfe2e2cf26ffa1e9f124","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146302601695232","occurred_at":"2026-09-24T15:35:15Z","failed_at":"provision","signature_id":"feaab49de56d93336deae269ed4dd6eb946386d142e5a85a65c80e15acb2b43d","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SwiftRecorder/management/deploy (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-24T16:33:44.590Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-09-24T16:33:45.122Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 values=\"map[captureMode:all clusterName:ci01-j1695232-mgmt-2 enabled:true environment:ci01 image:map[digest:sha256:21861cc03f51cd5c5fcc5f37f79aa945bfe633b1fbdb5c662d6376edb84a8a45 registry:arohcpsvcdev.azurecr.io repository:ci-op-s7p7jyyz/pipeline] region:canadacentral resources:map[limits:map[memory:256Mi] requests:map[cpu:20m memory:64Mi]] serviceAccount:map[name:swift-recorder]]\"\ntime=2026-09-24T16:33:45.122Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2\ntime=2026-09-24T16:33:45.293Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 namespace=swift-recorder\ntime=2026-09-24T16:33:45.465Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 dryRun=false\ntime=2026-09-24T16:33:45.465Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2\ntime=2026-09-24T16:33:45.516Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2\ntime=2026-09-24T16:43:47.867Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 err=\"resource DaemonSet/swift-recorder/swift-recorder not ready. status: InProgress, message: Available: 16/17\\ncontext deadline exceeded\"\ntime=2026-09-24T16:43:47.867Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/swift-recorder/swift-recorder not ready. status: InProgress, message: Available: 16/17\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d78f890a8b842ac794308d3c81fec10ddb32f68783d012d98fa9ac7172f3fde6","failure_pattern":"failed to roll out Helm release: resource DaemonSet/swift-recorder/swift-recorder not ready. status: InProgress, message: Available: 16/17 context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/swift-recorder/swift-recorder not ready. status: InProgress, message: Available: 16/17\\ncontext deadline exceeded"}},{"row_id":"d70418d34cbac9ff21bad1ee2bb85251a91c93da21db423d0f2ce6927a2680e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146302601695232","occurred_at":"2026-09-24T15:35:15Z","failed_at":"alert","signature_id":"7d7d5f484ae9cf25602860243ff6931ef9c6211bb6bd55469a3f901426372f60","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T16:29:24Z\n Ended: 2026-09-24T16:38:26Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j1695232-mgmt-1\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","d78f890a8b842ac794308d3c81fec10ddb32f68783d012d98fa9ac7172f3fde6"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103145551821279232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6917,"pr_state":"closed","pr_sha":"393527983bbf3810fedd8ca66cc790d3d1526978","final_merged_sha":"393527983bbf3810fedd8ca66cc790d3d1526978","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T15:32:28Z","started_at":"2026-09-24T15:32:28Z","completed_at":"2026-09-24T18:01:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T18:10:15.388329878Z","timing_metadata_checked_at":"2026-09-24T18:10:15.388329878Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T18:10:15.391069469Z","region_metadata_checked_at":"2026-09-24T18:10:15.391069469Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"b0bbded490a0c13c97fc2d1927db4cb84115a0d58690576619015f06aa8db678","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103145551821279232","occurred_at":"2026-09-24T15:32:28Z","failed_at":"alert","signature_id":"8c94a4b03784c995aa979e22c654512f8c6707bcd9d456af76a36f2b9dd1e313","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T17:49:20Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j1279232-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-5zn5h\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-5zn5h (namespace arobit) on cluster ci01-j1279232-svc has averaged 125.1% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7078/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103140357591011328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7078,"pr_state":"closed","pr_sha":"e39b04466ce928ac723794d88310cbc91d1b333d","final_merged_sha":"e39b04466ce928ac723794d88310cbc91d1b333d","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-24T15:11:50Z","started_at":"2026-09-24T15:11:50Z","completed_at":"2026-09-24T17:30:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T17:37:56.897245608Z","timing_metadata_checked_at":"2026-09-24T17:37:56.897245608Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T17:37:56.900599897Z","region_metadata_checked_at":"2026-09-24T17:37:56.900599897Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103139552272060416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7053,"pr_state":"closed","pr_sha":"b938b99cc7c5a09bd230dfa6d470fb9e6a8d6e0a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T15:08:38Z","started_at":"2026-09-24T15:08:38Z","completed_at":"2026-09-24T17:18:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T17:27:41.860715195Z","timing_metadata_checked_at":"2026-09-24T17:27:41.860715195Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T17:27:41.864661182Z","region_metadata_checked_at":"2026-09-24T17:27:41.864661182Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"a9ddbccb3b2cb71d92d6c5818417dcaaad1f119151f01a0ee3a7f3bab3e651c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103139552272060416","occurred_at":"2026-09-24T15:08:38Z","failed_at":"alert","signature_id":"2d8869925bade65a292b27f5efb8dda6ad2b9c0f4a99be4072145040c21d0bd5","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T17:03:20Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j2060416-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-6xqsv\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-6xqsv (namespace arobit) on cluster ci01-j2060416-svc has averaged 125.5% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7127/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103139542373502976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7127,"pr_state":"closed","pr_sha":"bea59940844633f6745b120a7fa79a808ab86c8d","final_merged_sha":"bea59940844633f6745b120a7fa79a808ab86c8d","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-24T15:08:35Z","started_at":"2026-09-24T15:08:35Z","completed_at":"2026-09-24T17:26:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T17:37:57.04799932Z","timing_metadata_checked_at":"2026-09-24T17:37:57.04799932Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T17:37:57.050639112Z","region_metadata_checked_at":"2026-09-24T17:37:57.050639112Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103136210296770560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7135,"pr_state":"closed","pr_sha":"78d2e7cdedac95816c4452d91dfcae35ac93f66a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T14:55:21Z","started_at":"2026-09-24T14:55:21Z","completed_at":"2026-09-24T17:12:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T17:16:45.938692147Z","timing_metadata_checked_at":"2026-09-24T17:16:45.938692147Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T17:16:45.941911536Z","region_metadata_checked_at":"2026-09-24T17:16:45.941911536Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"40c93b34062524f6a98381116ce1df2ecfecbdffcc8842272356ec92d3d1d301","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103136210296770560","occurred_at":"2026-09-24T14:55:21Z","failed_at":"alert","signature_id":"ec42ed0291695c354e24001557eb7c5d81b9f3854081433267b5026166de670e","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T16:55:52Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j6770560-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-fq4rt\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-fq4rt (namespace arobit) on cluster ci01-j6770560-svc has averaged 125.2% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"547cceba8cba4a1335b0db44aca4447a7a3245039be677b2101b75708ba40469","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103136210296770560","occurred_at":"2026-09-24T14:55:21Z","failed_at":"alert","signature_id":"95d616d62fd8b68affd2cc0528b1726d4f7bac8f448ad84828096223e0ed6a1a","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T16:32:06Z\n Ended: 2026-09-24T16:38:05Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6770560-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6770560-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"89fe6cd6e098ccfc071f8b9dc2d562c17fc4a7e5d0c4fb12bfdcf9d4abde7392","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103136210296770560","occurred_at":"2026-09-24T14:55:21Z","failed_at":"e2e","signature_id":"69e234e12ff1dfef8a8e49d1251b6e3da1b82338b95055a0f148ecb1c35fafd2","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create customer resources for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ac8540\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-autoscaling-hcp-cluster-pjz7p4\" in resourcegroup=\"autoscaling-cluster-7qnnh4q5h8fk\" to finish: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create customer resources for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ac8540\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-autoscaling-hcp-cluster-pjz7p4\" in resourcegroup=\"autoscaling-cluster-7qnnh4q5h8fk\" to finish: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"58103f0bb0f23a6bcc7180fb82d4fd105caee89b2c2667152f6ec564d76dd64c","failure_pattern":"failed to create customer-infra: failed waiting for deployment \"customer-infra-autoscaling-hcp-cluster-\u003cid\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish: context deadline exceeded","search_query":"failed to create customer-infra: failed waiting for deployment \"customer-infra-autoscaling-hcp-cluster-pjz7p4\" in resourcegroup=\"autoscaling-cluster-7qnnh4q5h8fk\" to finish: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["58103f0bb0f23a6bcc7180fb82d4fd105caee89b2c2667152f6ec564d76dd64c","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7136/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103133192167886848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7136,"pr_state":"closed","pr_sha":"43150589f42d359127b4be45d0ab0b12497df327","final_merged_sha":"43150589f42d359127b4be45d0ab0b12497df327","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-24T14:43:21Z","started_at":"2026-09-24T14:43:21Z","completed_at":"2026-09-24T16:58:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T17:06:33.598527208Z","timing_metadata_checked_at":"2026-09-24T17:06:33.598527208Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T17:06:33.606191579Z","region_metadata_checked_at":"2026-09-24T17:06:33.606191579Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103132311112388608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6687,"pr_state":"closed","pr_sha":"1769127263098290ba69f6f4e9bc0696f1c66e33","final_merged_sha":"969fb59d00e72634fd03a9f90aa105a57f5aef4b","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-24T14:39:51Z","started_at":"2026-09-24T14:39:51Z","completed_at":"2026-09-24T16:46:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T16:55:50.369860548Z","timing_metadata_checked_at":"2026-09-24T16:55:50.369860548Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T16:55:50.372807338Z","region_metadata_checked_at":"2026-09-24T16:55:50.372807338Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103126923872833536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6577,"pr_state":"closed","pr_sha":"d39d96bac8fc08b0445b52af3a342139074f812e","final_merged_sha":"3d03a40d19bf9b85b85165f4f9432cbbcf37480e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T14:18:18Z","started_at":"2026-09-24T14:18:18Z","completed_at":"2026-09-24T16:36:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T16:45:13.496190437Z","timing_metadata_checked_at":"2026-09-24T16:45:13.496190437Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T16:45:13.499519525Z","region_metadata_checked_at":"2026-09-24T16:45:13.499519525Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"67f9980f1230160bbdc632c6839dbe9ab51268d0d1b03ff69ba55acab4c280d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103126923872833536","occurred_at":"2026-09-24T14:18:18Z","failed_at":"e2e","signature_id":"001bfebb218903b1b7737fc9b09415cf50b63b5c1c904d8a608f8a8de2c5cb07","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00141d0c8\u003e: \n VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.33) failed: no version in clusterversion/version status.history is greater than initial \"4.21.33\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00141d0c8\u003e: \n VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.33) failed: no version in clusterversion/version status.history is greater than initial \"4.21.33\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4"}},{"row_id":"b82463f45bf52cef22d4cbc10ce1e4a5143edfb2f2e0b933c1447e42f9591088","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103126923872833536","occurred_at":"2026-09-24T14:18:18Z","failed_at":"alert","signature_id":"42836fa06f25aa02442410f2c7556dd0019e006d1c588c77355505f2e0955532","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T15:19:32Z\n Ended: 2026-09-24T16:14:33Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j2833536-svc\", component=\"backend\", name=\"controlplaneversionstatuscollector\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue controlplaneversionstatuscollector has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired"}},{"row_id":"ff9c016ab74a4107d3fa7a4e88e74b772174fe3982778c785228259a96a9a3e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103126923872833536","occurred_at":"2026-09-24T14:18:18Z","failed_at":"e2e","signature_id":"e31b9c3e430438e0c3a30a2dbdd5700e039ffeab1240ef87b983e42b9cff5173","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000e85578\u003e: \n VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.22.14) failed: no version in clusterversion/version status.history is greater than initial \"4.22.14\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000e85578\u003e: \n VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.22.14) failed: no version in clusterversion/version status.history is greater than initial \"4.22.14\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7134/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103126676954157056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7134,"pr_state":"closed","pr_sha":"dae8e35c480790849740550d6d82c953783fd1b2","final_merged_sha":"dae8e35c480790849740550d6d82c953783fd1b2","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-24T14:17:28Z","started_at":"2026-09-24T14:17:28Z","completed_at":"2026-09-24T16:35:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T16:45:13.63759746Z","timing_metadata_checked_at":"2026-09-24T16:45:13.63759746Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T16:45:13.64049005Z","region_metadata_checked_at":"2026-09-24T16:45:13.64049005Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103114369016532992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6915,"pr_state":"open","pr_sha":"e207b76e733cb65c677c16c7f38706c45baff4cd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T13:28:34Z","started_at":"2026-09-24T13:28:34Z","completed_at":"2026-09-24T15:52:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T16:03:30.771379679Z","timing_metadata_checked_at":"2026-09-24T16:03:30.771379679Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T16:03:30.775020667Z","region_metadata_checked_at":"2026-09-24T16:03:30.775020667Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"65d943e298058ab3dbe01ad97c7f43b6bad61ddce6e421684818bc75d678370e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103114369016532992","occurred_at":"2026-09-24T13:28:34Z","failed_at":"alert","signature_id":"017b8f139e649921e31c93332e3a78fedbb3f1194cc2d5d6b68bc998ac21643b","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T15:29:45Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j6532992-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-qkhfn\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-qkhfn (namespace arobit) on cluster ci01-j6532992-svc has averaged 125.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7130/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103111106493943808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7130,"pr_state":"closed","pr_sha":"8b117a5321fa82d95903dd74ddf66560af88c52e","final_merged_sha":"8b117a5321fa82d95903dd74ddf66560af88c52e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T13:15:33Z","started_at":"2026-09-24T13:15:33Z","completed_at":"2026-09-24T16:00:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T16:03:30.916061288Z","timing_metadata_checked_at":"2026-09-24T16:03:30.916061288Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T16:03:30.919086277Z","region_metadata_checked_at":"2026-09-24T16:03:30.919086277Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"57630b287ff5d17183136c1fc69c392417841504612c6a1add15ecc5f258b730","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7130/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103111106493943808","occurred_at":"2026-09-24T13:15:33Z","failed_at":"alert","signature_id":"71628d4204794d2bf5a1e409f017e03485e53450d5f8ef544bb7d4efc8d8ed17","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T15:02:27Z\n Ended: 2026-09-24T15:08:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3943808-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t1ecumio2t0so7i2erb9l3i7dm6lu3m-sre-hcp-cluster\", pod=\"kube-apiserver-5977bd7447-4zvrr\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t1ecumio2t0so7i2erb9l3i7dm6lu3m-sre-hcp-cluster/kube-apiserver-5977bd7447-4zvrr has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7131/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103110260871925760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7131,"pr_state":"closed","pr_sha":"cb8c2640a77a75ae501a0a609a7518ef1807d76b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T13:12:14Z","started_at":"2026-09-24T13:12:14Z","completed_at":"2026-09-24T15:53:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T16:03:31.019650336Z","timing_metadata_checked_at":"2026-09-24T16:03:31.019650336Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T16:03:31.022123727Z","region_metadata_checked_at":"2026-09-24T16:03:31.022123727Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"3e6ac057adb4d37c43d397e7f2da20997d166db47299cd8d563c34af8c4ed775","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7131/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103110260871925760","occurred_at":"2026-09-24T13:12:14Z","failed_at":"alert","signature_id":"0316ed16bae2b06726cffc3f37180891ba349c08b7cf18aa3d8d47e6021122fd","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T15:06:14Z\n Ended: 2026-09-24T15:12:18Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1925760-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1925760-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"cdd3f6111fd895c35f9c37397595036269d870dd6d0a997d723a640f74468ebd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7131/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103110260871925760","occurred_at":"2026-09-24T13:12:14Z","failed_at":"alert","signature_id":"db67d276ff0371dcc22947883bb69cbbfdb27c1787f10ba7322cfaa3b73aa198","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T15:28:31Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j1925760-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-fbnrq\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-fbnrq (namespace arobit) on cluster ci01-j1925760-svc has averaged 123.9% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103109521260941312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6746,"pr_state":"open","pr_sha":"841cb5dae9b7e324cc68cf77830e86bd32c2aaec","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T13:09:15Z","started_at":"2026-09-24T13:09:15Z","completed_at":"2026-09-24T15:55:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T16:03:30.611502523Z","timing_metadata_checked_at":"2026-09-24T16:03:30.611502523Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T16:03:30.615785208Z","region_metadata_checked_at":"2026-09-24T16:03:30.615785208Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"99e83122d756ead490fb985df2a87ed1b5286f4ca379b5f7f295f87e58a1e78c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103109521260941312","occurred_at":"2026-09-24T13:09:15Z","failed_at":"alert","signature_id":"d8ab59ad81a765023a17dd370421f6eb9d9e6dcc676911949deb1b2a6ad616d7","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T14:26:59Z\n Ended: 2026-09-24T15:00:00Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0941312-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0941312-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103106328183443456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7132,"pr_state":"open","pr_sha":"e21704ff18fc34470d5c68a223ff4b51b708a93f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T12:56:36Z","started_at":"2026-09-24T12:56:36Z","completed_at":"2026-09-24T15:03:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T15:20:35.809246393Z","timing_metadata_checked_at":"2026-09-24T15:20:35.809246393Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T15:20:35.812965681Z","region_metadata_checked_at":"2026-09-24T15:20:35.812965681Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"8a4dc582ace2964a17e02b2c75027d14758f5376c7d5a47f176b8b41ffdaa7ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103106328183443456","occurred_at":"2026-09-24T12:56:36Z","failed_at":"alert","signature_id":"03aacfb93bf8d8b6783be4fea0e1b48e62869181b58bb5f7f4cdf2c172610e39","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T14:47:52Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j3443456-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-grg4s\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-grg4s (namespace arobit) on cluster ci01-j3443456-svc has averaged 126.4% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103099181529829376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6917,"pr_state":"closed","pr_sha":"393527983bbf3810fedd8ca66cc790d3d1526978","final_merged_sha":"393527983bbf3810fedd8ca66cc790d3d1526978","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T12:28:13Z","started_at":"2026-09-24T12:28:13Z","completed_at":"2026-09-24T14:50:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T14:59:26.803675353Z","timing_metadata_checked_at":"2026-09-24T14:59:26.803675353Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T14:59:26.808162438Z","region_metadata_checked_at":"2026-09-24T14:59:26.808162438Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"69880ac9f7040e4ab836f767c139c5f0ecc5bcc61fd833877c7458ff5c06923a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103099181529829376","occurred_at":"2026-09-24T12:28:13Z","failed_at":"alert","signature_id":"ae74f769fa6d33550badb8679083a5b953a0ef29b38b87aaf9b9ee5f82f844cd","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T14:32:06Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j9829376-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-wm6g7\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-wm6g7 (namespace arobit) on cluster ci01-j9829376-svc has averaged 124.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103099182641319936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"closed","pr_sha":"87e40a07142a1457ed3f83374cc5d085c87d8f29","final_merged_sha":"e772682c6ada7bf3fd5674946d72fe25f79d834b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T12:28:13Z","started_at":"2026-09-24T12:28:13Z","completed_at":"2026-09-24T14:51:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T14:59:26.960253246Z","timing_metadata_checked_at":"2026-09-24T14:59:26.960253246Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T14:59:26.963621836Z","region_metadata_checked_at":"2026-09-24T14:59:26.963621836Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"36842f4c8c75043857d9d4b1fae459189dcb3352f9e8e571fe34311dc5b4d978","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103099182641319936","occurred_at":"2026-09-24T12:28:13Z","failed_at":"alert","signature_id":"598f96ff4cc7f38b9158674f984f82130937ad37ec6425753f1589dfdb918be5","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T14:32:41Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j1319936-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-z57ms\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-z57ms (namespace arobit) on cluster ci01-j1319936-svc has averaged 124.9% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103099159404875776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7112,"pr_state":"open","pr_sha":"395e45d0bbde804a9c22a549e978e9371ff65d91","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T12:28:07Z","started_at":"2026-09-24T12:28:07Z","completed_at":"2026-09-24T13:33:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T13:45:21.39251401Z","timing_metadata_checked_at":"2026-09-24T13:45:21.39251401Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T13:45:21.395868699Z","region_metadata_checked_at":"2026-09-24T13:45:21.395868699Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"1c6a7ad1a89532c58d6c1fb85414455d652c37f510eb604dd81c780a607bbc30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103099159404875776","occurred_at":"2026-09-24T12:28:07Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103099155617419264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6226,"pr_state":"closed","pr_sha":"9b50264285a60d3666e3df261a54ae027b4564e9","final_merged_sha":"9b50264285a60d3666e3df261a54ae027b4564e9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T12:28:06Z","started_at":"2026-09-24T12:28:06Z","completed_at":"2026-09-24T14:54:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T14:59:26.616215059Z","timing_metadata_checked_at":"2026-09-24T14:59:26.616215059Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T14:59:26.62220364Z","region_metadata_checked_at":"2026-09-24T14:59:26.62220364Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b01de8651f965d70aa6eb1ee99d6c8ec65d657f313df5007827fdd2e8a9ef19f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103099155617419264","occurred_at":"2026-09-24T12:28:06Z","failed_at":"e2e","signature_id":"d8b50df50945eaa6be746f392a278fe85b30b5ed045a7c3106d19fdebdab862c","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node k8w2j3t8n4o7w6y-cordon-np-fqrz9-2zs4k\nUnexpected error:\n \u003c*errors.StatusError | 0xc0012d1400\u003e: \n Operation cannot be fulfilled on nodes \"k8w2j3t8n4o7w6y-cordon-np-fqrz9-2zs4k\": the object has been modified; please apply your changes to the latest version and try again\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node k8w2j3t8n4o7w6y-cordon-np-fqrz9-2zs4k\nUnexpected error:\n \u003c*errors.StatusError | 0xc0012d1400\u003e: \n Operation cannot be fulfilled on nodes \"k8w2j3t8n4o7w6y-cordon-np-fqrz9-2zs4k\": the object has been modified; please apply your changes to the latest version and try again\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"750410b055b83de5666e72039e788f64cc42e64b3191a432c4cf3efab4dc693f","failure_pattern":"Operation cannot be fulfilled on nodes \"\u003cnode\u003e\": the object has been modified; please apply your changes to the latest version and try again","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["750410b055b83de5666e72039e788f64cc42e64b3191a432c4cf3efab4dc693f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103098516569067520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7065,"pr_state":"open","pr_sha":"c7822dfae8c930f61e17c2b06bf6a731cc37cb5d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T12:25:34Z","started_at":"2026-09-24T12:25:34Z","completed_at":"2026-09-24T14:54:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T14:59:27.15080043Z","timing_metadata_checked_at":"2026-09-24T14:59:27.15080043Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T14:59:27.153769021Z","region_metadata_checked_at":"2026-09-24T14:59:27.153769021Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"271ee1b51b4074d7f6655ef7ff30aa9ada8ea725c33a04a71345de901ef94abc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103098516569067520","occurred_at":"2026-09-24T12:25:34Z","failed_at":"e2e","signature_id":"ed02ea736a12b453c1efc5a8d4a86dc6bf8f47885f359a94f77192dc2ef4318b","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000721c20\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-8nk2s8\" in resourcegroup=\"rg-np-noedge-8nk2s8-pbf4w4lpxflj\" to finish creating: GET https://rp.j9067520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf579354-82a6-4511-b470-a74098b9d7f0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf579354-82a6-4511-b470-a74098b9d7f0\",\n \"name\": \"bf579354-82a6-4511-b470-a74098b9d7f0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T13:38:51.627589072Z\",\n \"endTime\": \"2026-09-24T13:57:59.764961372Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000721c20\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-8nk2s8\" in resourcegroup=\"rg-np-noedge-8nk2s8-pbf4w4lpxflj\" to finish creating: GET https://rp.j9067520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf579354-82a6-4511-b470-a74098b9d7f0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf579354-82a6-4511-b470-a74098b9d7f0\",\n \"name\": \"bf579354-82a6-4511-b470-a74098b9d7f0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T13:38:51.627589072Z\",\n \"endTime\": \"2026-09-24T13:57:59.764961372Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103089013568049152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7123,"pr_state":"closed","pr_sha":"c07c99eb845db32bdffea8c5c152b60fa095f319","final_merged_sha":"c07c99eb845db32bdffea8c5c152b60fa095f319","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T11:47:48Z","started_at":"2026-09-24T11:47:48Z","completed_at":"2026-09-24T14:16:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T14:37:23.934774819Z","timing_metadata_checked_at":"2026-09-24T14:37:23.934774819Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T14:37:23.938890706Z","region_metadata_checked_at":"2026-09-24T14:37:23.938890706Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"518bf9a7e970cec27034770e03dd884c932186a2d073041d0651b0dffd5f4ff3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103089013568049152","occurred_at":"2026-09-24T11:47:48Z","failed_at":"alert","signature_id":"ac2d659b6d8c225f448dfa59b70e7e1449c341473cabe70fbc8cfa0120d5269c","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T13:44:52Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j8049152-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-xmxv8\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-xmxv8 (namespace arobit) on cluster ci01-j8049152-svc has averaged 126.5% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"959a5145de23ce5b6a526b50df398e5391bea48f661b371ad30e155fda889a9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103089013568049152","occurred_at":"2026-09-24T11:47:48Z","failed_at":"e2e","signature_id":"a1699dd27afd8a16f801afb5b59631643f3ad5b14b803812cf5d901f58fe2d87","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005d4e40\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-mdj8lgwtvhhv\" to finish creating: GET https://rp.j8049152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1e1d081b-a194-4d7e-ad25-c5f0f75190c7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1e1d081b-a194-4d7e-ad25-c5f0f75190c7\",\n \"name\": \"1e1d081b-a194-4d7e-ad25-c5f0f75190c7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T12:47:20.580135273Z\",\n \"endTime\": \"2026-09-24T13:16:21.048177334Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 1m5s ago; version 4.21.34 is Partial (want Completed), started 3m10s ago; version 4.21.34 is Partial (want Completed), started 6m2s ago; version 4.21.34 is Partial (want Completed), started 6m38s ago; version 4.21.34 is Partial (want Completed), started 10m34s ago; version 4.21.34 is Partial (want Completed), started 16m16s ago; version 4.21.34 is Partial (want Completed), started 20m17s ago; version 4.21.34 is Partial (want Completed), started 23m52s ago; version 4.21.34 is Partial (want Completed), started 24m53s ago; version 4.21.34 is Partial (want Completed), started 25m36s ago; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005d4e40\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-mdj8lgwtvhhv\" to finish creating: GET https://rp.j8049152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1e1d081b-a194-4d7e-ad25-c5f0f75190c7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1e1d081b-a194-4d7e-ad25-c5f0f75190c7\",\n \"name\": \"1e1d081b-a194-4d7e-ad25-c5f0f75190c7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T12:47:20.580135273Z\",\n \"endTime\": \"2026-09-24T13:16:21.048177334Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 1m5s ago; version 4.21.34 is Partial (want Completed), started 3m10s ago; version 4.21.34 is Partial (want Completed), started 6m2s ago; version 4.21.34 is Partial (want Completed), started 6m38s ago; version 4.21.34 is Partial (want Completed), started 10m34s ago; version 4.21.34 is Partial (want Completed), started 16m16s ago; version 4.21.34 is Partial (want Completed), started 20m17s ago; version 4.21.34 is Partial (want Completed), started 23m52s ago; version 4.21.34 is Partial (want Completed), started 24m53s ago; version 4.21.34 is Partial (want Completed), started 25m36s ago; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0e33bf9c876122b5f27225338e1ee7dd193ea167c268da2c880fb16fb083cfb4","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator, openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["0e33bf9c876122b5f27225338e1ee7dd193ea167c268da2c880fb16fb083cfb4","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103088386418937856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6889,"pr_state":"open","pr_sha":"a20acc5249be416c7de2c6314cf69b79f4fdb00d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T11:45:17Z","started_at":"2026-09-24T11:45:17Z","completed_at":"2026-09-24T12:39:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T12:43:20.255180812Z","timing_metadata_checked_at":"2026-09-24T12:43:20.255180812Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T12:43:20.258652001Z","region_metadata_checked_at":"2026-09-24T12:43:20.258652001Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ba33de2680bd852986601c7682d66d111fbff5fa9f3ef00cfe053b1a848a1b34","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103088386418937856","occurred_at":"2026-09-24T11:45:17Z","failed_at":"provision","signature_id":"062e807a8092da1711f0e05e391e8b306673f16936facbff876328c71872d5ee","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/global/mirror-image","test_suite":"templatize-pipeline","failure_text":"time=2026-09-24T12:31:58.312Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e clustersService.image.registry \u003cnil\u003e}:{\u003cnil\u003e clustersService.image.repository \u003cnil\u003e}@{\u003cnil\u003e clustersService.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-09-24T12:32:36.173Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nFetch pull secret for source registry arohcpsvcdev from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image arohcpsvcdev/nshneor/cs@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 to arohcpsvcdev.azurecr.io/nshneor/cs:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6.\\nThe image will still be available under it's original digest sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 in the target registry.\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 arohcpsvcdev.azurecr.io/nshneor/cs:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 --from-registry-config /tmp/tmp.wiDcX7URTm/containers/auth.json --to-registry-config /tmp/tmp.wiDcX7URTm/containers/auth.json\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36601573516c72370b5de3f2f5199840f1883a8e5ad5f6d84f833bef7eb8e57a","failure_pattern":"Command failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:\u003chex\u003e arohcpsvcdev.azurecr.io/nshneor/cs:\u003chex\u003e --from-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json --to-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json","search_query":"Command failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 arohcpsvcdev.azurecr.io/nshneor/cs:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 --from-registry-config /tmp/tmp.bxHO86cpwB/containers/auth.json --to-registry-config /tmp/tmp.bxHO86cpwB/containers/auth.json"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36601573516c72370b5de3f2f5199840f1883a8e5ad5f6d84f833bef7eb8e57a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103078971544440832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7009,"pr_state":"closed","pr_sha":"30ab7aab2c6cd7fcf1f6520d0a43a948520ff9af","final_merged_sha":"c83bdabaf08bdb32938b4526f3e12a94c26f811c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T11:07:54Z","started_at":"2026-09-24T11:07:54Z","completed_at":"2026-09-24T13:20:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T13:24:39.348907499Z","timing_metadata_checked_at":"2026-09-24T13:24:39.348907499Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T13:24:39.351709291Z","region_metadata_checked_at":"2026-09-24T13:24:39.351709291Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"68c8ae01d980f4fff587f887ae3a1831b755e92f87fbf488762eeae660f1d0cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103078971544440832","occurred_at":"2026-09-24T11:07:54Z","failed_at":"alert","signature_id":"c3c8df3e62c73e7b4c4be31e24a070ec366f9237969b5d0bdd9fb689ac18eaa2","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T13:04:24Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j4440832-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-tgfp5\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-tgfp5 (namespace arobit) on cluster ci01-j4440832-svc has averaged 126.2% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"c24f72d47de592656e453f693fec0e34c3ee697523833690bbd669abf718afcc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103078971544440832","occurred_at":"2026-09-24T11:07:54Z","failed_at":"alert","signature_id":"52a5a38b47cc6e69825210c368c8f32d8cf5f29e6283dbea51644a315c099800","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T12:40:11Z\n Ended: 2026-09-24T12:46:14Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j4440832-svc\", component=\"frontend\", method=\"post\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/requestadmincredential\", severity=\"warning\"\n Description: More than 5% of frontend requests for post /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/requestadmincredential exceeded 1 second over the past 30 minutes (cluster ci01-j4440832-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"ef9f9a88a9e37855451ce9a79689d947f7274ceaa210d62e60bba6b46ae7dff7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103078971544440832","occurred_at":"2026-09-24T11:07:54Z","failed_at":"e2e","signature_id":"b935a8a8e1ec9f7c77dd3ec089fb77afdc37f7a2e866b79eb951cee11d3e6a47","test_name":"KSM HCP Metrics metrics should be present in HCP Azure Monitoring Workspace","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:185]: Timed out after 1500.000s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:183 with:\nexpected kube_node_status_condition from kube-state-metrics; kube_node_info from kube-state-metrics but got no results\nExpected\n \u003c[]string | len:2, cap:2\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:185]: Timed out after 1500.000s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:183 with:\nexpected kube_node_status_condition from kube-state-metrics; kube_node_info from kube-state-metrics but got no results\nExpected\n \u003c[]string | len:2, cap:2\u003e: ...\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d91796eeda5752e6e2b49b398ec435f90a4ab6d3394ea2d0cd4ce5b2c2f30a11","failure_pattern":"not all KSM metrics appeared in Azure Monitor","search_query":"not all KSM metrics appeared in Azure Monitor"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","d91796eeda5752e6e2b49b398ec435f90a4ab6d3394ea2d0cd4ce5b2c2f30a11","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103078389232439296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7045,"pr_state":"closed","pr_sha":"711d134f86fa2910df2819fd4922f71db33ae23c","final_merged_sha":"711d134f86fa2910df2819fd4922f71db33ae23c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T11:05:35Z","started_at":"2026-09-24T11:05:35Z","completed_at":"2026-09-24T13:16:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T13:24:39.495797998Z","timing_metadata_checked_at":"2026-09-24T13:24:39.495797998Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T13:24:39.499248388Z","region_metadata_checked_at":"2026-09-24T13:24:39.499248388Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3483eae33fbf52f687ba3d182a15ece722c63dcd6cd184726f947449e7567b32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103078389232439296","occurred_at":"2026-09-24T11:05:35Z","failed_at":"e2e","signature_id":"90dd6825e45727c7dbdbd1c4c8cd0b287a1d641727762971db09d4ac1b8c9c2a","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001cb6930\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001cb6930\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103073435138658304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7117,"pr_state":"closed","pr_sha":"ea43f7ba6df9f089661d7c30a490eb1e026104e9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-24T10:45:54Z","started_at":"2026-09-24T10:45:54Z","completed_at":"2026-09-24T12:58:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T13:04:07.27398292Z","timing_metadata_checked_at":"2026-09-24T13:04:07.27398292Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T13:04:07.277753109Z","region_metadata_checked_at":"2026-09-24T13:04:07.277753109Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103070229281116160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6995,"pr_state":"closed","pr_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","final_merged_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-24T10:33:10Z","started_at":"2026-09-24T10:33:10Z","completed_at":"2026-09-24T12:41:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T12:43:20.422473688Z","timing_metadata_checked_at":"2026-09-24T12:43:20.422473688Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T12:43:20.425118979Z","region_metadata_checked_at":"2026-09-24T12:43:20.425118979Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103063632513339392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7074,"pr_state":"open","pr_sha":"1d4696a7f321f969159f69c5b6bbb66c06a7afda","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-24T10:06:57Z","started_at":"2026-09-24T10:06:57Z","completed_at":"2026-09-24T12:24:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T12:32:24.746862683Z","timing_metadata_checked_at":"2026-09-24T12:32:24.746862683Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T12:32:24.749779476Z","region_metadata_checked_at":"2026-09-24T12:32:24.749779476Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103060333542248448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6889,"pr_state":"open","pr_sha":"a20acc5249be416c7de2c6314cf69b79f4fdb00d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T09:53:51Z","started_at":"2026-09-24T09:53:51Z","completed_at":"2026-09-24T10:39:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T10:47:49.531619118Z","timing_metadata_checked_at":"2026-09-24T10:47:49.531619118Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T10:47:49.537901198Z","region_metadata_checked_at":"2026-09-24T10:47:49.537901198Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b61e4db1d32f0c8619d89299fa81ab50088869bedf7fcad06e1fd0cb3e18e12e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103060333542248448","occurred_at":"2026-09-24T09:53:51Z","failed_at":"provision","signature_id":"062e807a8092da1711f0e05e391e8b306673f16936facbff876328c71872d5ee","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/global/mirror-image","test_suite":"templatize-pipeline","failure_text":"time=2026-09-24T10:30:31.503Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e clustersService.image.registry \u003cnil\u003e}:{\u003cnil\u003e clustersService.image.repository \u003cnil\u003e}@{\u003cnil\u003e clustersService.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-09-24T10:31:04.673Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nFetch pull secret for source registry arohcpsvcdev from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image arohcpsvcdev/nshneor/cs@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 to arohcpsvcdev.azurecr.io/nshneor/cs:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6.\\nThe image will still be available under it's original digest sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 in the target registry.\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 arohcpsvcdev.azurecr.io/nshneor/cs:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 --from-registry-config /tmp/tmp.bxHO86cpwB/containers/auth.json --to-registry-config /tmp/tmp.bxHO86cpwB/containers/auth.json\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36601573516c72370b5de3f2f5199840f1883a8e5ad5f6d84f833bef7eb8e57a","failure_pattern":"Command failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:\u003chex\u003e arohcpsvcdev.azurecr.io/nshneor/cs:\u003chex\u003e --from-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json --to-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json","search_query":"Command failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 arohcpsvcdev.azurecr.io/nshneor/cs:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 --from-registry-config /tmp/tmp.bxHO86cpwB/containers/auth.json --to-registry-config /tmp/tmp.bxHO86cpwB/containers/auth.json"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36601573516c72370b5de3f2f5199840f1883a8e5ad5f6d84f833bef7eb8e57a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103059223502917632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6886,"pr_state":"closed","pr_sha":"a6aa6a03a137ec55e3f4f5d06cc620e6bf6dd7f3","final_merged_sha":"a6aa6a03a137ec55e3f4f5d06cc620e6bf6dd7f3","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-24T09:49:26Z","started_at":"2026-09-24T09:49:26Z","completed_at":"2026-09-24T12:00:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T12:11:35.1015385Z","timing_metadata_checked_at":"2026-09-24T12:11:35.1015385Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T12:11:35.10453219Z","region_metadata_checked_at":"2026-09-24T12:11:35.10453219Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055146278719488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"closed","pr_sha":"87e40a07142a1457ed3f83374cc5d085c87d8f29","final_merged_sha":"e772682c6ada7bf3fd5674946d72fe25f79d834b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T09:33:14Z","started_at":"2026-09-24T09:33:14Z","completed_at":"2026-09-24T11:54:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T12:00:58.983893576Z","timing_metadata_checked_at":"2026-09-24T12:00:58.983893576Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T12:00:58.986814067Z","region_metadata_checked_at":"2026-09-24T12:00:58.986814067Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"30196de459fe9d4ab659367c5a72ec384a55e85cf98834fc65698aedf6b9df42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055146278719488","occurred_at":"2026-09-24T09:33:14Z","failed_at":"alert","signature_id":"a410a024ffc0f26ed41523481fe4553235341e68f41df89e12a0df8d70a45e2c","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T11:22:06Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j8719488-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-k4tb8\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-k4tb8 (namespace arobit) on cluster ci01-j8719488-svc has averaged 125.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055129497309184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6917,"pr_state":"closed","pr_sha":"393527983bbf3810fedd8ca66cc790d3d1526978","final_merged_sha":"393527983bbf3810fedd8ca66cc790d3d1526978","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T09:33:10Z","started_at":"2026-09-24T09:33:10Z","completed_at":"2026-09-24T11:39:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T11:50:35.256020056Z","timing_metadata_checked_at":"2026-09-24T11:50:35.256020056Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T11:50:35.258387748Z","region_metadata_checked_at":"2026-09-24T11:50:35.258387748Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"1ad3e381f0887f9dde1548be5b33025754578f12a2547eff126153dec8d5fdf8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055129497309184","occurred_at":"2026-09-24T09:33:10Z","failed_at":"e2e","signature_id":"dd690b63f5f18c2dbee25e78c6edf3eeb01364dc7ce7e0d6fe2be002e3e85b36","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f01380\u003e: \n failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-bpr946thzk52\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/62555631-7ec6-4b02-b164-698c4d6f90b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/62555631-7ec6-4b02-b164-698c4d6f90b4\",\n \"name\": \"62555631-7ec6-4b02-b164-698c4d6f90b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T10:24:31.78038551Z\",\n \"endTime\": \"2026-09-24T10:25:09.920364245Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f01380\u003e: \n failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-bpr946thzk52\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/62555631-7ec6-4b02-b164-698c4d6f90b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/62555631-7ec6-4b02-b164-698c4d6f90b4\",\n \"name\": \"62555631-7ec6-4b02-b164-698c4d6f90b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T10:24:31.78038551Z\",\n \"endTime\": \"2026-09-24T10:25:09.920364245Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"50137feb992408c4d63cf98263eefe9617c109cdad6aebd129e45849ebda0176","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055129497309184","occurred_at":"2026-09-24T09:33:10Z","failed_at":"alert","signature_id":"fecdb95305d7912b34e85ac60e3a333a4105e66f05be7ed59027fc3150684b84","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T11:20:27Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j7309184-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-klrjj\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-klrjj (namespace arobit) on cluster ci01-j7309184-svc has averaged 123.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"afe50feacbb965aab0db973be2e241576aa3fc116f71ee7d60c831518a5a8961","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055129497309184","occurred_at":"2026-09-24T09:33:10Z","failed_at":"e2e","signature_id":"c83a023c6700a1a72267fb7de4ac02824e0ac5dfe0406b81242145d8b6fc24f6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-d4v522\" for cluster \"cluster-candidate-4-22-d4v522\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008ad540\u003e: \n failed to create NodePool np-d4v522: failed waiting for nodepool=\"np-d4v522\" for cluster \"cluster-candidate-4-22-d4v522\" in resourcegroup=\"rg-candidate-4-22-q2jqvkjfll66\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a57495d-192b-47f3-8f08-3dbd939c2ccc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a57495d-192b-47f3-8f08-3dbd939c2ccc\",\n \"name\": \"9a57495d-192b-47f3-8f08-3dbd939c2ccc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T10:30:51.31445342Z\",\n \"endTime\": \"2026-09-24T10:31:30.134898766Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-d4v522\" for cluster \"cluster-candidate-4-22-d4v522\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008ad540\u003e: \n failed to create NodePool np-d4v522: failed waiting for nodepool=\"np-d4v522\" for cluster \"cluster-candidate-4-22-d4v522\" in resourcegroup=\"rg-candidate-4-22-q2jqvkjfll66\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a57495d-192b-47f3-8f08-3dbd939c2ccc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a57495d-192b-47f3-8f08-3dbd939c2ccc\",\n \"name\": \"9a57495d-192b-47f3-8f08-3dbd939c2ccc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T10:30:51.31445342Z\",\n \"endTime\": \"2026-09-24T10:31:30.134898766Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b0bee23e3e4cbf37d3bbc8806af1cdda6852d6d35b35a246694097f0f5626e55","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055129497309184","occurred_at":"2026-09-24T09:33:10Z","failed_at":"e2e","signature_id":"12c0bbf57d474e0cd6cf2451ff2cf882646f3b861a1cce2aae346ddfcc9bb6c4","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gqsdx8vgs4cm\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/020af38d-4caa-4310-a595-881bec9e9405\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/020af38d-4caa-4310-a595-881bec9e9405\",\n \"name\": \"020af38d-4caa-4310-a595-881bec9e9405\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T10:30:43.520242825Z\",\n \"endTime\": \"2026-09-24T10:31:20.235327414Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n}\n--------------------------------------------------------------------------------\n failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gqsdx8vgs4cm\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edfe1588-77de-4f6d-a724-94f5c5b56313\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edfe1588-77de-4f6d-a724-94f5c5b56313\",\n \"name\": \"edfe1588-77de-4f6d-a724-94f5c5b56313\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T10:30:43.575931644Z\",\n \"endTime\": \"2026-09-24T10:31:20.247783059Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n \u003c[]error | len:2, cap:2\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gqsdx8vgs4cm\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/020af38d-4caa-4310-a595-881bec9e9405\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/020af38d-4caa-4310-a595-881bec9e9405\",\n \"name\": \"020af38d-4caa-4310-a595-881bec9e9405\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T10:30:43.520242825Z\",\n \"endTime\": \"2026-09-24T10:31:20.235327414Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n}\n--------------------------------------------------------------------------------\n failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gqsdx8vgs4cm\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edfe1588-77de-4f6d-a724-94f5c5b56313\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edfe1588-77de-4f6d-a724-94f5c5b56313\",\n \"name\": \"edfe1588-77de-4f6d-a724-94f5c5b56313\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T10:30:43.575931644Z\",\n \"endTime\": \"2026-09-24T10:31:20.247783059Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n \u003c[]error | len:2, cap:2\u003e: ...\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5","failure_pattern":"--------------------------------------------------------------------------------","search_query":"--------------------------------------------------------------------------------"}},{"row_id":"d0e0c47ab370af3c06f20703974bd81c6300e1829fffd3dcb8b9c4c10db73033","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055129497309184","occurred_at":"2026-09-24T09:33:10Z","failed_at":"e2e","signature_id":"648b322dd2b99b65149b365370cd6cb7208f3487446dfc081728eb5b19b547de","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a31f40\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-dlxzmd\" in resourcegroup=\"rg-np-skip-minor-dlxzmd-8tb8jspnqx48\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2e06418b-9c11-492d-8bf1-bc7db3b3afdf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2e06418b-9c11-492d-8bf1-bc7db3b3afdf\",\n \"name\": \"2e06418b-9c11-492d-8bf1-bc7db3b3afdf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T10:30:31.115048125Z\",\n \"endTime\": \"2026-09-24T10:31:10.228268414Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a31f40\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-dlxzmd\" in resourcegroup=\"rg-np-skip-minor-dlxzmd-8tb8jspnqx48\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2e06418b-9c11-492d-8bf1-bc7db3b3afdf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2e06418b-9c11-492d-8bf1-bc7db3b3afdf\",\n \"name\": \"2e06418b-9c11-492d-8bf1-bc7db3b3afdf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T10:30:31.115048125Z\",\n \"endTime\": \"2026-09-24T10:31:10.228268414Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ed93db6fa4d1246a0d20969055291a0a3ee5820bcf9daedfff524dc8fa867c9a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055129497309184","occurred_at":"2026-09-24T09:33:10Z","failed_at":"alert","signature_id":"6dc0e8968ec60f0324b6d07caa91d4f95c59a07d57bde431637fa99a19424678","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T10:49:49Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7309184-svc\", component=\"frontend\", method=\"get\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} has exceeded 250ms over the past 30 minutes (cluster ci01-j7309184-svc).\nFiring 2:\n State: Resolved\n Started: 2026-09-24T10:54:48Z\n Ended: 2026-09-24T11:00:53Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7309184-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7309184-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055124225069056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6226,"pr_state":"closed","pr_sha":"9b50264285a60d3666e3df261a54ae027b4564e9","final_merged_sha":"9b50264285a60d3666e3df261a54ae027b4564e9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T09:33:09Z","started_at":"2026-09-24T09:33:09Z","completed_at":"2026-09-24T11:42:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T11:50:34.961805525Z","timing_metadata_checked_at":"2026-09-24T11:50:34.961805525Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T11:50:34.965190114Z","region_metadata_checked_at":"2026-09-24T11:50:34.965190114Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"eb01da93a50ccc0e719bd731dced0957d81dbc45c621e09216e8a9bc05d8b398","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055124225069056","occurred_at":"2026-09-24T09:33:09Z","failed_at":"alert","signature_id":"94cc4e8f43733aad08243e0c9d7614fcd959bf692829e56a21d66324fe6568f8","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T11:31:37Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j5069056-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-qdd5r\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-qdd5r (namespace arobit) on cluster ci01-j5069056-svc has averaged 123.7% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7127/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055116536909824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7127,"pr_state":"closed","pr_sha":"1931f7632f6505326b075b7bf4af50e95992b83f","final_merged_sha":"bea59940844633f6745b120a7fa79a808ab86c8d","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T09:33:07Z","started_at":"2026-09-24T09:33:07Z","completed_at":"2026-09-24T11:53:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T12:01:00.005704849Z","timing_metadata_checked_at":"2026-09-24T12:01:00.005704849Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T12:01:00.008250341Z","region_metadata_checked_at":"2026-09-24T12:01:00.008250341Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"724e2e57062b3b179a842d6e24b29818d5eb1001f07c315e048e64160125333a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7127/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055116536909824","occurred_at":"2026-09-24T09:33:07Z","failed_at":"alert","signature_id":"33cdaea8b871495d558c36e8c9add34f9b20b7366192a7363dd1ee244293bb4b","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T11:07:54Z\n Ended: 2026-09-24T11:15:54Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j6909824-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-757775d6c9-5zxj7\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-757775d6c9-5zxj7 (namespace kube-applier) on cluster ci01-j6909824-mgmt-1 has averaged 122.3% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-24T11:35:51Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j6909824-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-cf5qk\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-cf5qk (namespace arobit) on cluster ci01-j6909824-svc has averaged 125% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055091593383936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7112,"pr_state":"open","pr_sha":"395e45d0bbde804a9c22a549e978e9371ff65d91","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T09:33:01Z","started_at":"2026-09-24T09:33:01Z","completed_at":"2026-09-24T11:53:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T12:00:59.290054739Z","timing_metadata_checked_at":"2026-09-24T12:00:59.290054739Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T12:00:59.29296743Z","region_metadata_checked_at":"2026-09-24T12:00:59.29296743Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"198d78a9113a4e78ea5039f291e871b4d308fa3acf54b4cf8844f664e90f954a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055091593383936","occurred_at":"2026-09-24T09:33:01Z","failed_at":"alert","signature_id":"f048e9c9961e95f1fc58ff19153a9940b10cb95928a512cbddafab4912d29e0d","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T10:54:07Z\n Ended: 2026-09-24T11:02:08Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3383936-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3383936-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103034733737807872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7045,"pr_state":"closed","pr_sha":"711d134f86fa2910df2819fd4922f71db33ae23c","final_merged_sha":"711d134f86fa2910df2819fd4922f71db33ae23c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T08:12:03Z","started_at":"2026-09-24T08:12:03Z","completed_at":"2026-09-24T10:47:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T10:57:50.925202504Z","timing_metadata_checked_at":"2026-09-24T10:57:50.925202504Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T10:57:50.928085695Z","region_metadata_checked_at":"2026-09-24T10:57:50.928085695Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"6651b8f48d30faff929448f391901fa40e1b038949f92992699965cfb999f205","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103034733737807872","occurred_at":"2026-09-24T08:12:03Z","failed_at":"alert","signature_id":"34737877aa3d834f2668fa205d349b5946f34479dc5d4c773b3f919ae25587c8","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T09:55:30Z\n Ended: 2026-09-24T10:12:28Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j7807872-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-54b4fcc545-sxchr\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-54b4fcc545-sxchr (namespace kube-applier) on cluster ci01-j7807872-mgmt-1 has averaged 131.6% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-24T10:33:26Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j7807872-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-w94xl\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-w94xl (namespace arobit) on cluster ci01-j7807872-svc has averaged 124.3% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103026030053691392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7009,"pr_state":"closed","pr_sha":"d3a0a892c526ec1fe6a58902acc6005994742c36","final_merged_sha":"c83bdabaf08bdb32938b4526f3e12a94c26f811c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T07:37:32Z","started_at":"2026-09-24T07:37:32Z","completed_at":"2026-09-24T09:45:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T09:56:34.555754492Z","timing_metadata_checked_at":"2026-09-24T09:56:34.555754492Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T09:56:34.558307584Z","region_metadata_checked_at":"2026-09-24T09:56:34.558307584Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0c7a6514027a90afb6930171d85d5fe27f0525f4693ce0dfac2c94f24fe38c6d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103026030053691392","occurred_at":"2026-09-24T07:37:32Z","failed_at":"e2e","signature_id":"b9e992e4dedaba952b861c4393e5971bfc00933bdfd510734b71bce7ea9324b7","test_name":"KSM HCP Metrics metrics should be present in HCP Azure Monitoring Workspace","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:147]: Timed out after 900.000s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:143 with:\nexpected ingresscontroller_info metrics from kube-state-metrics but got no results\nExpected\n \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:147]: Timed out after 900.000s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:143 with:\nexpected ingresscontroller_info metrics from kube-state-metrics but got no results\nExpected\n \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d91796eeda5752e6e2b49b398ec435f90a4ab6d3394ea2d0cd4ce5b2c2f30a11","failure_pattern":"not all KSM metrics appeared in Azure Monitor","search_query":"not all KSM metrics appeared in Azure Monitor"}},{"row_id":"6d18b20f0786113148cb7b27d22bef8acdec4fc12a7b318af02d7250ccb3aceb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103026030053691392","occurred_at":"2026-09-24T07:37:32Z","failed_at":"alert","signature_id":"967796441e8b38c863ade49bdd491e7af8db03f90c559c543dbfafc769074a03","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T09:32:10Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j3691392-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-dnmk5\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-dnmk5 (namespace arobit) on cluster ci01-j3691392-svc has averaged 128.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["d91796eeda5752e6e2b49b398ec435f90a4ab6d3394ea2d0cd4ce5b2c2f30a11","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7071/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103025067402203136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7071,"pr_state":"open","pr_sha":"901730f14fbdd5486107a4de254e41a92ff32f55","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-24T07:33:42Z","started_at":"2026-09-24T07:33:42Z","completed_at":"2026-09-24T10:00:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T10:06:43.631902449Z","timing_metadata_checked_at":"2026-09-24T10:06:43.631902449Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T10:06:43.635257639Z","region_metadata_checked_at":"2026-09-24T10:06:43.635257639Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103024914473684992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6995,"pr_state":"closed","pr_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","final_merged_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T07:33:06Z","started_at":"2026-09-24T07:33:06Z","completed_at":"2026-09-24T09:48:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T09:56:34.403953984Z","timing_metadata_checked_at":"2026-09-24T09:56:34.403953984Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T09:56:34.406776474Z","region_metadata_checked_at":"2026-09-24T09:56:34.406776474Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"04236b81cdfb088971ef993d269c570bdf0726f013da351d8515f94586b642a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103024914473684992","occurred_at":"2026-09-24T07:33:06Z","failed_at":"e2e","signature_id":"569706336b0bb772a90b6de148018cee0d7066e56c9777657f620d7bb7c75db6","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100c160\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-wjm85h57dqlz\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/deca3434-569c-4feb-abcd-1a0dae06f921\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/deca3434-569c-4feb-abcd-1a0dae06f921\",\n \"name\": \"deca3434-569c-4feb-abcd-1a0dae06f921\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T08:38:16.231035723Z\",\n \"endTime\": \"2026-09-24T08:57:18.997473969Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100c160\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-wjm85h57dqlz\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/deca3434-569c-4feb-abcd-1a0dae06f921\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/deca3434-569c-4feb-abcd-1a0dae06f921\",\n \"name\": \"deca3434-569c-4feb-abcd-1a0dae06f921\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T08:38:16.231035723Z\",\n \"endTime\": \"2026-09-24T08:57:18.997473969Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0585177b7def4349955b91c214bd9b23b0335aa0c3b8aa4954daed1c5e31300f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103024914473684992","occurred_at":"2026-09-24T07:33:06Z","failed_at":"e2e","signature_id":"28954fd4b540505310c8d2b6c0bea86f4588ec03f99fb14067897e28bef073a8","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:214]: failed to cordon node a2u7a3d0r9z7j0c-cordon-np-tvtlh-5pbth\nUnexpected error:\n \u003c*errors.StatusError | 0xc000a01cc0\u003e: \n Operation cannot be fulfilled on nodes \"a2u7a3d0r9z7j0c-cordon-np-tvtlh-5pbth\": the object has been modified; please apply your changes to the latest version and try again\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:214]: failed to cordon node a2u7a3d0r9z7j0c-cordon-np-tvtlh-5pbth\nUnexpected error:\n \u003c*errors.StatusError | 0xc000a01cc0\u003e: \n Operation cannot be fulfilled on nodes \"a2u7a3d0r9z7j0c-cordon-np-tvtlh-5pbth\": the object has been modified; please apply your changes to the latest version and try again\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"750410b055b83de5666e72039e788f64cc42e64b3191a432c4cf3efab4dc693f","failure_pattern":"Operation cannot be fulfilled on nodes \"\u003cnode\u003e\": the object has been modified; please apply your changes to the latest version and try again","search_query":"occurred"}},{"row_id":"b7cef400d0b6f1a1fab0063cac273fb1ef934efe823505e5bb8d9f7e24975c00","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103024914473684992","occurred_at":"2026-09-24T07:33:06Z","failed_at":"alert","signature_id":"774298937099e086350da5d45672b54735082f9b7780d4691b1ecf885eb8f21d","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T09:06:52Z\n Ended: 2026-09-24T09:12:54Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j3684992-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-6c8d85cd9b-rllfq\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-6c8d85cd9b-rllfq (namespace kube-applier) on cluster ci01-j3684992-mgmt-1 has averaged 120.9% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["750410b055b83de5666e72039e788f64cc42e64b3191a432c4cf3efab4dc693f","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103015428132442112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6884,"pr_state":"open","pr_sha":"50a683fcdea603a7e5aec2dbc6a88e41a26babc3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T06:55:24Z","started_at":"2026-09-24T06:55:24Z","completed_at":"2026-09-24T09:07:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T09:13:58.682905317Z","timing_metadata_checked_at":"2026-09-24T09:13:58.682905317Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T09:13:58.685848907Z","region_metadata_checked_at":"2026-09-24T09:13:58.685848907Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"3b7a03a4c328850702365ab89e51f31eec453f3d147c055b528fe78ad942ccc2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103015428132442112","occurred_at":"2026-09-24T06:55:24Z","failed_at":"alert","signature_id":"609ef47bc200be1ff06713d78b2540ac62b150a8d0e17b016ed8c4fd89fa13fe","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T08:24:07Z\n Ended: 2026-09-24T08:31:08Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j2442112-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j2442112-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7127/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103015078914691072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7127,"pr_state":"closed","pr_sha":"1931f7632f6505326b075b7bf4af50e95992b83f","final_merged_sha":"bea59940844633f6745b120a7fa79a808ab86c8d","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T06:54:00Z","started_at":"2026-09-24T06:54:00Z","completed_at":"2026-09-24T09:02:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T09:13:59.020412116Z","timing_metadata_checked_at":"2026-09-24T09:13:59.020412116Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T09:13:59.023260507Z","region_metadata_checked_at":"2026-09-24T09:13:59.023260507Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"fa333532258cf72060cfdb8938ca4af40bdbf50e94b17e0570715bc4378ccb4a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7127/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103015078914691072","occurred_at":"2026-09-24T06:54:00Z","failed_at":"e2e","signature_id":"423b6e69577f5d18ef882789710f57eff74dad24bb4c0c9c09ca525154f2add8","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.004s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000bbd950\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.004s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000bbd950\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6503,"pr_state":"closed","pr_sha":"a0bf660bfdf32222374a82406e0907cae4b53869","final_merged_sha":"a0bf660bfdf32222374a82406e0907cae4b53869","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T06:52:47Z","started_at":"2026-09-24T06:52:47Z","completed_at":"2026-09-24T09:04:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T09:13:58.526504427Z","timing_metadata_checked_at":"2026-09-24T09:13:58.526504427Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T09:13:58.530245115Z","region_metadata_checked_at":"2026-09-24T09:13:58.530245115Z"},"failed_at":["alert","e2e"],"failed_test_count":51,"occurrences":[{"row_id":"0229d58cafed02811393b087cc1033fb8fa90a46d0eca25a4d2ae41d7bcf0553","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"9efffb68dfea6877253ddef458a441c37343a3bffa28ae1e9b9b91339b87e767","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-fmpd7w\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010bc760\u003e: \n failed to create HCP cluster np-noedge-fmpd7w: failed waiting for cluster=\"np-noedge-fmpd7w\" in resourcegroup=\"rg-np-noedge-fmpd7w-4wtwkhwbhdsv\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/adfaa1aa-c887-47ec-bd5c-47ba88c8592e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/adfaa1aa-c887-47ec-bd5c-47ba88c8592e\",\n \"name\": \"adfaa1aa-c887-47ec-bd5c-47ba88c8592e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:06.643774992Z\",\n \"endTime\": \"2026-09-24T08:10:12.46029994Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-fmpd7w\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010bc760\u003e: \n failed to create HCP cluster np-noedge-fmpd7w: failed waiting for cluster=\"np-noedge-fmpd7w\" in resourcegroup=\"rg-np-noedge-fmpd7w-4wtwkhwbhdsv\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/adfaa1aa-c887-47ec-bd5c-47ba88c8592e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/adfaa1aa-c887-47ec-bd5c-47ba88c8592e\",\n \"name\": \"adfaa1aa-c887-47ec-bd5c-47ba88c8592e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:06.643774992Z\",\n \"endTime\": \"2026-09-24T08:10:12.46029994Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0868b4908a036790c57172bb154c16bbf7b813bf7118f24ddf915b7faef9cf6d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"7418caf66506fff095427d27ff9a805df356854e0d321f2288eaa6d95d485b6b","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004728e0\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-8gr7g4hqsnhr\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07eb76f7-9cbf-4a37-b1b0-0105d67443e2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07eb76f7-9cbf-4a37-b1b0-0105d67443e2\",\n \"name\": \"07eb76f7-9cbf-4a37-b1b0-0105d67443e2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:43.4059014Z\",\n \"endTime\": \"2026-09-24T08:09:52.291702842Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004728e0\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-8gr7g4hqsnhr\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07eb76f7-9cbf-4a37-b1b0-0105d67443e2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07eb76f7-9cbf-4a37-b1b0-0105d67443e2\",\n \"name\": \"07eb76f7-9cbf-4a37-b1b0-0105d67443e2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:43.4059014Z\",\n \"endTime\": \"2026-09-24T08:09:52.291702842Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"12e203b46189b805b84aac25eaf0c1bf9e5d6a69e7d6672d617f02d06572ccd9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"f7d663f4b5351be1b0ab0e046173f64d43548235e11b8e908df07ebaa35514ad","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:91]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010b7380\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-9vsw8frrpj6b\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d84f7256-c730-4d44-b567-d707a70c8f66\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d84f7256-c730-4d44-b567-d707a70c8f66\",\n \"name\": \"d84f7256-c730-4d44-b567-d707a70c8f66\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:43.719680919Z\",\n \"endTime\": \"2026-09-24T08:10:52.46601694Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:91]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010b7380\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-9vsw8frrpj6b\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d84f7256-c730-4d44-b567-d707a70c8f66\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d84f7256-c730-4d44-b567-d707a70c8f66\",\n \"name\": \"d84f7256-c730-4d44-b567-d707a70c8f66\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:43.719680919Z\",\n \"endTime\": \"2026-09-24T08:10:52.46601694Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"131a7be2148fe5e30f61d9cb07ac56a62e239ec8a7265c0d7210bf2a7fc1604e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"b9b50758a1aa006036c7c1ee58af90cd509c7b1900722ef0229b46ed2f5e8563","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"150d6cb7e5ed310407a0f19bf9180b0704888983beb85e7d5f959613e6c2aa12","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"ff19a7ca5677fbfdea92fb5506f95dcdd516adf5a368b611a7ee13809b4061f3","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe31e0\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-qdtctgm6r2dh\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb29f891-c7c8-4f58-bafa-fe41c7962ab4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb29f891-c7c8-4f58-bafa-fe41c7962ab4\",\n \"name\": \"eb29f891-c7c8-4f58-bafa-fe41c7962ab4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:59.649072249Z\",\n \"endTime\": \"2026-09-24T08:10:02.528985242Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe31e0\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-qdtctgm6r2dh\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb29f891-c7c8-4f58-bafa-fe41c7962ab4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb29f891-c7c8-4f58-bafa-fe41c7962ab4\",\n \"name\": \"eb29f891-c7c8-4f58-bafa-fe41c7962ab4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:59.649072249Z\",\n \"endTime\": \"2026-09-24T08:10:02.528985242Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1cd777dcad712a09ddb3a9db48253fef408bbcaf2de42af347f8c370786e8d0f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"91bb81a2eca403f9cb3887462e7e35230600fa3a01fbacce487d699ab1f7ea14","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe2360\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-b9nd58zbz7q6\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95a52094-c367-4204-a2ba-3b032472550c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95a52094-c367-4204-a2ba-3b032472550c\",\n \"name\": \"95a52094-c367-4204-a2ba-3b032472550c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:42:11.309207669Z\",\n \"endTime\": \"2026-09-24T08:11:12.420425917Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe2360\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-b9nd58zbz7q6\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95a52094-c367-4204-a2ba-3b032472550c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95a52094-c367-4204-a2ba-3b032472550c\",\n \"name\": \"95a52094-c367-4204-a2ba-3b032472550c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:42:11.309207669Z\",\n \"endTime\": \"2026-09-24T08:11:12.420425917Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1d94e061b463a8605d443bd44a55dc1f553a47fb0e067647dffb2e9d5a1817ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"alert","signature_id":"dd3366bd92c2564cb6f449817e8c4d513e034b81a60fe1b01c561f9570ac3115","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T08:49:12Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"prometheus-collector\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8969088/providers/microsoft.monitor/accounts/services-j8969088\", namespace=\"kube-system\", pod=\"ama-metrics-node-5rpz7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"OOMKilled\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8afb0369-75a6-47c9-a399-fdc0457fc23d\"\n Description: Container prometheus-collector in pod kube-system/ama-metrics-node-5rpz7 on cluster ci01-j8969088-mgmt-1 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-24T08:50:13Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"prometheus-collector\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8969088/providers/microsoft.monitor/accounts/services-j8969088\", namespace=\"kube-system\", pod=\"ama-metrics-node-5rpz7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"OOMKilled\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8afb0369-75a6-47c9-a399-fdc0457fc23d\"\n Description: Container prometheus-collector in pod kube-system/ama-metrics-node-5rpz7 on cluster ci01-j8969088-mgmt-1 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c4e2bd93d0410f48ddfa390cc77604492607a58b2a4eaff204baa6c7bed3af07","failure_pattern":"alert [svc] KubeContainerOOMKilled fired","search_query":"alert [svc] KubeContainerOOMKilled fired"}},{"row_id":"1edbf4efaf111abf965c9d86ac0ba1b68510a2438973d0c6d93dc3cb8ea37bc3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"7517cba222e621154182cc4ff18e45927f0a4901a789bfab867f7abd52b27ff1","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005ba640\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-4qqgrzwnh6kq\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1fff1d81-05ea-4f1f-bc93-4e7c3046f5a8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1fff1d81-05ea-4f1f-bc93-4e7c3046f5a8\",\n \"name\": \"1fff1d81-05ea-4f1f-bc93-4e7c3046f5a8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:49.734246907Z\",\n \"endTime\": \"2026-09-24T08:09:52.355812866Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005ba640\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-4qqgrzwnh6kq\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1fff1d81-05ea-4f1f-bc93-4e7c3046f5a8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1fff1d81-05ea-4f1f-bc93-4e7c3046f5a8\",\n \"name\": \"1fff1d81-05ea-4f1f-bc93-4e7c3046f5a8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:49.734246907Z\",\n \"endTime\": \"2026-09-24T08:09:52.355812866Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1ee9f76f84f5dd3fecc341e7d8759464743cf851cbeb8afcb3f0186aef445aba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"61d2a995fa7cf5627014299f798acd76615bfafedc8979ad3e36a0eb0c7a2662","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-bbpc2z\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016105a0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-bbpc2z: failed waiting for cluster=\"cp-ystream-upgrade-4-22-bbpc2z\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-bjzg5zftb8c7\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b1d0e542-cab2-4c4d-963a-e4b5fd1adb9a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b1d0e542-cab2-4c4d-963a-e4b5fd1adb9a\",\n \"name\": \"b1d0e542-cab2-4c4d-963a-e4b5fd1adb9a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:57.986337274Z\",\n \"endTime\": \"2026-09-24T08:10:02.774914631Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-bbpc2z\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016105a0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-bbpc2z: failed waiting for cluster=\"cp-ystream-upgrade-4-22-bbpc2z\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-bjzg5zftb8c7\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b1d0e542-cab2-4c4d-963a-e4b5fd1adb9a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b1d0e542-cab2-4c4d-963a-e4b5fd1adb9a\",\n \"name\": \"b1d0e542-cab2-4c4d-963a-e4b5fd1adb9a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:57.986337274Z\",\n \"endTime\": \"2026-09-24T08:10:02.774914631Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"24c85a7b6737df9fde20f4f55269e08be30b96f639009c7bf60e334c433158c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"502b4a1a6e0a45081dc28c470a10f7dc5d98fa3c12fe64214fb3ad5d8b1d6533","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-thsfhh9pwl4h/cluster-candidate-4-23-dgcrgd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b069a0\u003e: \n failed to create HCP cluster cluster-candidate-4-23-dgcrgd: failed waiting for cluster=\"cluster-candidate-4-23-dgcrgd\" in resourcegroup=\"rg-candidate-4-23-thsfhh9pwl4h\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/46f50ba0-17f4-4547-8b56-f7d1fe1bfc64\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/46f50ba0-17f4-4547-8b56-f7d1fe1bfc64\",\n \"name\": \"46f50ba0-17f4-4547-8b56-f7d1fe1bfc64\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:43:03.016197463Z\",\n \"endTime\": \"2026-09-24T08:12:12.583643302Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-thsfhh9pwl4h/cluster-candidate-4-23-dgcrgd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b069a0\u003e: \n failed to create HCP cluster cluster-candidate-4-23-dgcrgd: failed waiting for cluster=\"cluster-candidate-4-23-dgcrgd\" in resourcegroup=\"rg-candidate-4-23-thsfhh9pwl4h\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/46f50ba0-17f4-4547-8b56-f7d1fe1bfc64\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/46f50ba0-17f4-4547-8b56-f7d1fe1bfc64\",\n \"name\": \"46f50ba0-17f4-4547-8b56-f7d1fe1bfc64\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:43:03.016197463Z\",\n \"endTime\": \"2026-09-24T08:12:12.583643302Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"297be4c48b5b423ad0fc7ebb5e6bc65e44d0dca9e687bd540c20ef7ab62bc82e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"d92c446c7baf0ccb5ed48e5e01951aa4edeca26f9fddab2165a4455ffedb13f5","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-lk4kvk at version 4.22.15\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019efc0\u003e: \n failed to create HCP cluster np-skip-minor-lk4kvk: failed waiting for cluster=\"np-skip-minor-lk4kvk\" in resourcegroup=\"rg-np-skip-minor-lk4kvk-rbzvnhhflz9g\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21670f9e-8792-444f-ba20-e46f0f4b3ea3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21670f9e-8792-444f-ba20-e46f0f4b3ea3\",\n \"name\": \"21670f9e-8792-444f-ba20-e46f0f4b3ea3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:25.862776281Z\",\n \"endTime\": \"2026-09-24T08:10:32.989125032Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-lk4kvk at version 4.22.15\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019efc0\u003e: \n failed to create HCP cluster np-skip-minor-lk4kvk: failed waiting for cluster=\"np-skip-minor-lk4kvk\" in resourcegroup=\"rg-np-skip-minor-lk4kvk-rbzvnhhflz9g\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21670f9e-8792-444f-ba20-e46f0f4b3ea3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21670f9e-8792-444f-ba20-e46f0f4b3ea3\",\n \"name\": \"21670f9e-8792-444f-ba20-e46f0f4b3ea3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:25.862776281Z\",\n \"endTime\": \"2026-09-24T08:10:32.989125032Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2de9f22f71c5b00e78bf36111de71452ce8ca727699948e34b7d60aa06cbee02","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"1dbed21c30a383bd27b5dec9ae60753fd2bb2ee86f63bc372cce6f9506e12b4d","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001004be0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-gpcpkt74ppvw\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/587473b4-b0b4-4bac-af0c-59df402089ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/587473b4-b0b4-4bac-af0c-59df402089ed\",\n \"name\": \"587473b4-b0b4-4bac-af0c-59df402089ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:00.74741343Z\",\n \"endTime\": \"2026-09-24T08:10:02.836497296Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001004be0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-gpcpkt74ppvw\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/587473b4-b0b4-4bac-af0c-59df402089ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/587473b4-b0b4-4bac-af0c-59df402089ed\",\n \"name\": \"587473b4-b0b4-4bac-af0c-59df402089ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:00.74741343Z\",\n \"endTime\": \"2026-09-24T08:10:02.836497296Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"39a6587a7dc037433d2309c9db0add964fc398bcc6cfa738f84835acca1a06ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"c44a3318beb91865b30052009d15e7f5adae10dc8a6c1f345d1fd64c63b78b16","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00097d920\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-lmrsgcfq8bzh\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/533b7d1e-0bde-4626-b741-445b44e4a89d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/533b7d1e-0bde-4626-b741-445b44e4a89d\",\n \"name\": \"533b7d1e-0bde-4626-b741-445b44e4a89d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:05.940292708Z\",\n \"endTime\": \"2026-09-24T08:10:12.161250282Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00097d920\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-lmrsgcfq8bzh\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/533b7d1e-0bde-4626-b741-445b44e4a89d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/533b7d1e-0bde-4626-b741-445b44e4a89d\",\n \"name\": \"533b7d1e-0bde-4626-b741-445b44e4a89d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:05.940292708Z\",\n \"endTime\": \"2026-09-24T08:10:12.161250282Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3a33fe1dc23fbed43bcd648413a068bc7f4aee6dde9619bdd8be2b33573a0a1d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"6502c23f1e7ec0cf695a6476bed4b50f740a286a0b7fe2eed834a4003a061b59","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-bgb52s\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001750740\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-bgb52s: failed waiting for cluster=\"cp-ystream-upgrade-4-21-bgb52s\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-jcjftx6b9x2k\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fe4078d-119a-41a1-81dc-1a867d38ae04\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fe4078d-119a-41a1-81dc-1a867d38ae04\",\n \"name\": \"7fe4078d-119a-41a1-81dc-1a867d38ae04\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:43:20.810067266Z\",\n \"endTime\": \"2026-09-24T08:12:22.35611824Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-bgb52s\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001750740\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-bgb52s: failed waiting for cluster=\"cp-ystream-upgrade-4-21-bgb52s\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-jcjftx6b9x2k\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fe4078d-119a-41a1-81dc-1a867d38ae04\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fe4078d-119a-41a1-81dc-1a867d38ae04\",\n \"name\": \"7fe4078d-119a-41a1-81dc-1a867d38ae04\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:43:20.810067266Z\",\n \"endTime\": \"2026-09-24T08:12:22.35611824Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3b059793fe0ce52efda6378043946ad80a19f6bddabd627832a9993854cb7b15","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"4725982cb9ff6e3263bd7373f034c89fe01c51d46047009fa1ed4ed4ee62e38c","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001314120\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-wqd8qjsnv8xd\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b3ee8b94-682e-49ee-b356-7a598c2fe1d3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b3ee8b94-682e-49ee-b356-7a598c2fe1d3\",\n \"name\": \"b3ee8b94-682e-49ee-b356-7a598c2fe1d3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:43:29.385273559Z\",\n \"endTime\": \"2026-09-24T08:12:32.441059735Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001314120\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-wqd8qjsnv8xd\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b3ee8b94-682e-49ee-b356-7a598c2fe1d3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b3ee8b94-682e-49ee-b356-7a598c2fe1d3\",\n \"name\": \"b3ee8b94-682e-49ee-b356-7a598c2fe1d3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:43:29.385273559Z\",\n \"endTime\": \"2026-09-24T08:12:32.441059735Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"405f54a9c06dd6a86371ec2e4b4596643294cfdb8ed932d66350ab929b91d9a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"e4af5e82f7cb726f365dd5dbed7b27d574f194386873276eef8bf8579929b089","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011127e0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-8brn2zfrkbx8\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ba16c17-48e2-489f-8aab-24141a06ffe2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ba16c17-48e2-489f-8aab-24141a06ffe2\",\n \"name\": \"0ba16c17-48e2-489f-8aab-24141a06ffe2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:42:06.511933878Z\",\n \"endTime\": \"2026-09-24T08:11:12.384994979Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011127e0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-8brn2zfrkbx8\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ba16c17-48e2-489f-8aab-24141a06ffe2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ba16c17-48e2-489f-8aab-24141a06ffe2\",\n \"name\": \"0ba16c17-48e2-489f-8aab-24141a06ffe2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:42:06.511933878Z\",\n \"endTime\": \"2026-09-24T08:11:12.384994979Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4116abcfb517e5b3339a151262223c269304065d40ac9f95d328f72358f22baf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"9fd5e8c5641bc6751859b9a36c13659f53bb439f0e7a58e1cb5d575101bf0bb9","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ff7540\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-jd2dztkf4q6k\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e3b8117-db91-4910-a368-8430a162834b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e3b8117-db91-4910-a368-8430a162834b\",\n \"name\": \"7e3b8117-db91-4910-a368-8430a162834b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:10.256195992Z\",\n \"endTime\": \"2026-09-24T08:10:12.484682887Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ff7540\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-jd2dztkf4q6k\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e3b8117-db91-4910-a368-8430a162834b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e3b8117-db91-4910-a368-8430a162834b\",\n \"name\": \"7e3b8117-db91-4910-a368-8430a162834b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:10.256195992Z\",\n \"endTime\": \"2026-09-24T08:10:12.484682887Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4a3b8c9bb34d04c234fc15a3f32f2a7c026e14599a41dbc2844ccc91fac19114","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"e1aa6d1a4b4a58d1ebe60d873db4901be2203876d71301a5e963bfa6dd923550","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-bpwr8b with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00142a820\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-bpwr8b: failed waiting for cluster=\"np-version-upgrade-cluster-bpwr8b\" in resourcegroup=\"rg-np-version-upgrade-bpwr8b-5mspbx8tgwc9\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9312c512-9dd4-490e-99db-1b34b5d5cac7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9312c512-9dd4-490e-99db-1b34b5d5cac7\",\n \"name\": \"9312c512-9dd4-490e-99db-1b34b5d5cac7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:46.64988457Z\",\n \"endTime\": \"2026-09-24T08:10:52.356439556Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-bpwr8b with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00142a820\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-bpwr8b: failed waiting for cluster=\"np-version-upgrade-cluster-bpwr8b\" in resourcegroup=\"rg-np-version-upgrade-bpwr8b-5mspbx8tgwc9\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9312c512-9dd4-490e-99db-1b34b5d5cac7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9312c512-9dd4-490e-99db-1b34b5d5cac7\",\n \"name\": \"9312c512-9dd4-490e-99db-1b34b5d5cac7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:46.64988457Z\",\n \"endTime\": \"2026-09-24T08:10:52.356439556Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4ab2559745e2c11e17e51db7c03df0aed3b3df54e1f5d13c233e34b4b727fa66","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"1472d735a7d85501b8306e000dd5d8b6374ba7f340c2e4ceebc2265fef3c431f","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3eee0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-rg8hmhqgcfz9\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcb0cbaa-a0ca-4bbe-b03f-c5925421d183\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcb0cbaa-a0ca-4bbe-b03f-c5925421d183\",\n \"name\": \"bcb0cbaa-a0ca-4bbe-b03f-c5925421d183\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:42.824972244Z\",\n \"endTime\": \"2026-09-24T08:10:42.484532745Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3eee0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-rg8hmhqgcfz9\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcb0cbaa-a0ca-4bbe-b03f-c5925421d183\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcb0cbaa-a0ca-4bbe-b03f-c5925421d183\",\n \"name\": \"bcb0cbaa-a0ca-4bbe-b03f-c5925421d183\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:42.824972244Z\",\n \"endTime\": \"2026-09-24T08:10:42.484532745Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"59a4e012b563d12fce45c5bba7c0c704d3531f306b45b91ad9c846d8883313bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"91d967ad1e2051c89285c002d456dc98a5e8ae32206d5cccbef47bc1c326378e","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d20dc0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-5llh7g78vw6q\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c070108d-3dd7-4a82-b7a5-982d63a36ddb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c070108d-3dd7-4a82-b7a5-982d63a36ddb\",\n \"name\": \"c070108d-3dd7-4a82-b7a5-982d63a36ddb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:45:12.179449162Z\",\n \"endTime\": \"2026-09-24T08:14:13.110781488Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d20dc0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-5llh7g78vw6q\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c070108d-3dd7-4a82-b7a5-982d63a36ddb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c070108d-3dd7-4a82-b7a5-982d63a36ddb\",\n \"name\": \"c070108d-3dd7-4a82-b7a5-982d63a36ddb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:45:12.179449162Z\",\n \"endTime\": \"2026-09-24T08:14:13.110781488Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"619b134ae5618a319877981f2deffc8987b77653752bae2114f5d61f5ca1ee9a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"61d69de89a82efb1426dba6f56799b44a42e7adce363871a9a6a5cea8d8616fc","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00088b940\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-7l295vlxbdxb\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a91ca760-905a-453e-aece-d1f8c0a8a310\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a91ca760-905a-453e-aece-d1f8c0a8a310\",\n \"name\": \"a91ca760-905a-453e-aece-d1f8c0a8a310\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:49.345259416Z\",\n \"endTime\": \"2026-09-24T08:09:52.246509313Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00088b940\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-7l295vlxbdxb\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a91ca760-905a-453e-aece-d1f8c0a8a310\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a91ca760-905a-453e-aece-d1f8c0a8a310\",\n \"name\": \"a91ca760-905a-453e-aece-d1f8c0a8a310\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:49.345259416Z\",\n \"endTime\": \"2026-09-24T08:09:52.246509313Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"61f81a70ec6390dcf6a5e66f2064bb1325a3baa7ff8559e43c34a492d9cb5bdf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"79eab85227deaa5a504f26b49c3794ac1f2793daeeccfb151c2ebda50a94f291","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-8fwnnv\" with version 4.21.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ed2400\u003e: \n failed to create HCP cluster cluster-zstream-4-21-8fwnnv: failed waiting for cluster=\"cluster-zstream-4-21-8fwnnv\" in resourcegroup=\"rg-zstream-upgrade-4-21-vfhxf5lxkw26\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/72ee287d-f4bc-49c0-bd18-3c4bc5c28cf6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/72ee287d-f4bc-49c0-bd18-3c4bc5c28cf6\",\n \"name\": \"72ee287d-f4bc-49c0-bd18-3c4bc5c28cf6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:21.799680174Z\",\n \"endTime\": \"2026-09-24T08:10:22.395149597Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-8fwnnv\" with version 4.21.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ed2400\u003e: \n failed to create HCP cluster cluster-zstream-4-21-8fwnnv: failed waiting for cluster=\"cluster-zstream-4-21-8fwnnv\" in resourcegroup=\"rg-zstream-upgrade-4-21-vfhxf5lxkw26\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/72ee287d-f4bc-49c0-bd18-3c4bc5c28cf6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/72ee287d-f4bc-49c0-bd18-3c4bc5c28cf6\",\n \"name\": \"72ee287d-f4bc-49c0-bd18-3c4bc5c28cf6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:21.799680174Z\",\n \"endTime\": \"2026-09-24T08:10:22.395149597Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"642303ed9bd128883d2bfd6e6899e90fcf826c89012a29d9901e3a231dd6e198","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"114c59c2bf6570c0eefc1b7b5e5a65d9ac897e0594f2a59218c2788db88a3400","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-kzgnsh\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000be2600\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-kzgnsh: failed waiting for cluster=\"cp-ystream-upgrade-5-0-kzgnsh\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-nh8nv9nrhbfl\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e49bd5a2-59c3-457f-8f93-e961de30f26b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e49bd5a2-59c3-457f-8f93-e961de30f26b\",\n \"name\": \"e49bd5a2-59c3-457f-8f93-e961de30f26b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:06.715026226Z\",\n \"endTime\": \"2026-09-24T08:10:12.435086078Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-kzgnsh\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000be2600\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-kzgnsh: failed waiting for cluster=\"cp-ystream-upgrade-5-0-kzgnsh\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-nh8nv9nrhbfl\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e49bd5a2-59c3-457f-8f93-e961de30f26b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e49bd5a2-59c3-457f-8f93-e961de30f26b\",\n \"name\": \"e49bd5a2-59c3-457f-8f93-e961de30f26b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:06.715026226Z\",\n \"endTime\": \"2026-09-24T08:10:12.435086078Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"66b867a2b9a1607e3eb58428b3b389898310e7bb61f71d363f97f0b7e5b151e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"038937131ff2f8c58cc268033cbe88327c0ecf6260a7273ef6763e38ee56eae3","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000687d40\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-jl6ktm48b4wh\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a0fce08-88e2-4170-ac88-340cc76559e9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a0fce08-88e2-4170-ac88-340cc76559e9\",\n \"name\": \"1a0fce08-88e2-4170-ac88-340cc76559e9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:08.812973457Z\",\n \"endTime\": \"2026-09-24T08:10:12.41059037Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000687d40\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-jl6ktm48b4wh\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a0fce08-88e2-4170-ac88-340cc76559e9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a0fce08-88e2-4170-ac88-340cc76559e9\",\n \"name\": \"1a0fce08-88e2-4170-ac88-340cc76559e9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:08.812973457Z\",\n \"endTime\": \"2026-09-24T08:10:12.41059037Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"677a3d86f711ecdf983a9a2ffa0c2bf6005740f0dfd5c0583cde8084e2304a12","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"0da4df177339bd718ad09c5a87da0bc13c7840ad3a67b75bdb0840e9e8d78aac","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fced80\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-tmsknb268d6w\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03151659-c9c2-4800-86d8-c833276deaec\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03151659-c9c2-4800-86d8-c833276deaec\",\n \"name\": \"03151659-c9c2-4800-86d8-c833276deaec\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:08.321205783Z\",\n \"endTime\": \"2026-09-24T08:10:12.392342627Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fced80\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-tmsknb268d6w\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03151659-c9c2-4800-86d8-c833276deaec\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03151659-c9c2-4800-86d8-c833276deaec\",\n \"name\": \"03151659-c9c2-4800-86d8-c833276deaec\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:08.321205783Z\",\n \"endTime\": \"2026-09-24T08:10:12.392342627Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"709b5cb17c632ab8cb60a5453ea11947b4e423d9680707b47f70b5b89014a7c6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"f1c7a583a370723cfe1f78b08a12ea6d5d2e3ceec2dcab1ff58a6c2cc20ec68f","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005a6820\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-rrzl9k8rbh7d\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49663312-fd2f-4b2f-9deb-ff772ef15618\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49663312-fd2f-4b2f-9deb-ff772ef15618\",\n \"name\": \"49663312-fd2f-4b2f-9deb-ff772ef15618\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:44.029242055Z\",\n \"endTime\": \"2026-09-24T08:10:52.338714145Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005a6820\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-rrzl9k8rbh7d\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49663312-fd2f-4b2f-9deb-ff772ef15618\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/49663312-fd2f-4b2f-9deb-ff772ef15618\",\n \"name\": \"49663312-fd2f-4b2f-9deb-ff772ef15618\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:44.029242055Z\",\n \"endTime\": \"2026-09-24T08:10:52.338714145Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"724ff8dc23e27b720128b8c48ecc6c0315596a19f9acd1f223e0fac3353fbb2e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"878621b3a630c1b8725c92562eb8fd6ff7e73ca262fd7d2d1a4d3c703bd7181b","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00105e440\u003e: \n failed to create HCP cluster ea-listfc6z95: failed waiting for cluster=\"ea-listfc6z95\" in resourcegroup=\"ea-list-5wp4zk6r4rrw\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5700361a-8406-4ed2-b5d9-0451a549ba2a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5700361a-8406-4ed2-b5d9-0451a549ba2a\",\n \"name\": \"5700361a-8406-4ed2-b5d9-0451a549ba2a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:57.144125452Z\",\n \"endTime\": \"2026-09-24T08:10:02.732206911Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00105e440\u003e: \n failed to create HCP cluster ea-listfc6z95: failed waiting for cluster=\"ea-listfc6z95\" in resourcegroup=\"ea-list-5wp4zk6r4rrw\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5700361a-8406-4ed2-b5d9-0451a549ba2a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5700361a-8406-4ed2-b5d9-0451a549ba2a\",\n \"name\": \"5700361a-8406-4ed2-b5d9-0451a549ba2a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:57.144125452Z\",\n \"endTime\": \"2026-09-24T08:10:02.732206911Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7b486a2f0e6992b9e21d1d60f50a80818641192faeb6ad009e0f70c39cdc9f0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"3ae4d5da8c6399c11ae57e63b6c3ccd31dbb7427337ebb04cd7c8172656b71c1","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-kvwp8j\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a94c40\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-kvwp8j: failed waiting for cluster=\"cp-ystream-upgrade-4-23-kvwp8j\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-4vh2m6c88cvz\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f6113a1-441b-4312-b307-177259b6598e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f6113a1-441b-4312-b307-177259b6598e\",\n \"name\": \"7f6113a1-441b-4312-b307-177259b6598e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:43:22.445406408Z\",\n \"endTime\": \"2026-09-24T08:12:22.485453604Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-kvwp8j\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a94c40\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-kvwp8j: failed waiting for cluster=\"cp-ystream-upgrade-4-23-kvwp8j\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-4vh2m6c88cvz\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f6113a1-441b-4312-b307-177259b6598e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f6113a1-441b-4312-b307-177259b6598e\",\n \"name\": \"7f6113a1-441b-4312-b307-177259b6598e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:43:22.445406408Z\",\n \"endTime\": \"2026-09-24T08:12:22.485453604Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7e5d165e26db7bd19e2f16115c96b88c0d02765173de3ff41cf22af8ebcb01ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"c5761bbbe7076b8cad854fc72af7546a6a999f18ed1f68b4bec3edea52bf3259","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-ns5vxp with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001053020\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-ns5vxp: failed waiting for cluster=\"np-version-upgrade-cluster-ns5vxp\" in resourcegroup=\"rg-np-version-upgrade-ns5vxp-qlk2d8k64jmc\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/66762dec-8bf6-4481-96f8-9eab1b8fe4b7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/66762dec-8bf6-4481-96f8-9eab1b8fe4b7\",\n \"name\": \"66762dec-8bf6-4481-96f8-9eab1b8fe4b7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:43:39.9635078Z\",\n \"endTime\": \"2026-09-24T08:12:42.211710402Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-ns5vxp with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001053020\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-ns5vxp: failed waiting for cluster=\"np-version-upgrade-cluster-ns5vxp\" in resourcegroup=\"rg-np-version-upgrade-ns5vxp-qlk2d8k64jmc\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/66762dec-8bf6-4481-96f8-9eab1b8fe4b7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/66762dec-8bf6-4481-96f8-9eab1b8fe4b7\",\n \"name\": \"66762dec-8bf6-4481-96f8-9eab1b8fe4b7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:43:39.9635078Z\",\n \"endTime\": \"2026-09-24T08:12:42.211710402Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7f7c6eab398d5e9119ffa44fdf21427539b4d2a7660eb0adfbe5b4953e230245","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"7c3d27901cce4d1113cefa72f1c0a7197bf58cec686d39a74e26062de2c2d826","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a47580\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-vzvn5vwv5v84\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db53e723-4940-4e66-b894-c7e7a08246c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db53e723-4940-4e66-b894-c7e7a08246c4\",\n \"name\": \"db53e723-4940-4e66-b894-c7e7a08246c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:45:12.276028787Z\",\n \"endTime\": \"2026-09-24T08:14:12.888632144Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a47580\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-vzvn5vwv5v84\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db53e723-4940-4e66-b894-c7e7a08246c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db53e723-4940-4e66-b894-c7e7a08246c4\",\n \"name\": \"db53e723-4940-4e66-b894-c7e7a08246c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:45:12.276028787Z\",\n \"endTime\": \"2026-09-24T08:14:12.888632144Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"86deedbd4c1f50652f13503fa8c7fc8f58168b98b25072088d3d09989a598125","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"35ebec740614a91af05434491fc4858f3e451d08e674d21eef134a37cf5d6508","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00177b7a0\u003e: \n GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9f1663ab-23a4-497c-9d66-58443957086c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9f1663ab-23a4-497c-9d66-58443957086c\",\n \"name\": \"9f1663ab-23a4-497c-9d66-58443957086c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:42:33.894126591Z\",\n \"endTime\": \"2026-09-24T08:11:42.316112022Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00177b7a0\u003e: \n GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9f1663ab-23a4-497c-9d66-58443957086c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9f1663ab-23a4-497c-9d66-58443957086c\",\n \"name\": \"9f1663ab-23a4-497c-9d66-58443957086c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:42:33.894126591Z\",\n \"endTime\": \"2026-09-24T08:11:42.316112022Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"914e39535ed68c159ea7ecf262880b2d14c56be0469a34fdc57343073f2a0a4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"55c4b969417e538731e376035caad7977a5aca6f1f2c72913b4936de93c287e7","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-q4rkxv\" with version 4.22.14 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00109a3a0\u003e: \n failed to create HCP cluster cluster-zstream-4-22-q4rkxv: failed waiting for cluster=\"cluster-zstream-4-22-q4rkxv\" in resourcegroup=\"rg-zstream-upgrade-4-22-rsclzz4n7xrr\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1b2698d5-fbaa-4027-bd59-ee008fcc812d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1b2698d5-fbaa-4027-bd59-ee008fcc812d\",\n \"name\": \"1b2698d5-fbaa-4027-bd59-ee008fcc812d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:55.905410994Z\",\n \"endTime\": \"2026-09-24T08:10:02.839433414Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-q4rkxv\" with version 4.22.14 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00109a3a0\u003e: \n failed to create HCP cluster cluster-zstream-4-22-q4rkxv: failed waiting for cluster=\"cluster-zstream-4-22-q4rkxv\" in resourcegroup=\"rg-zstream-upgrade-4-22-rsclzz4n7xrr\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1b2698d5-fbaa-4027-bd59-ee008fcc812d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1b2698d5-fbaa-4027-bd59-ee008fcc812d\",\n \"name\": \"1b2698d5-fbaa-4027-bd59-ee008fcc812d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:55.905410994Z\",\n \"endTime\": \"2026-09-24T08:10:02.839433414Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"92d186ac654a3dc34cd26668b23d30dd3ef6a1bd3e427422f228e3b4a365a009","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"8f058c62c69f94c523da555f854a97a9bc6956f04a0436c7a2ee43ab827dddb2","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8e8a0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-k9w78gnprtl2\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f599fa04-67e1-455d-a292-28baa109ae91\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f599fa04-67e1-455d-a292-28baa109ae91\",\n \"name\": \"f599fa04-67e1-455d-a292-28baa109ae91\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:46.285678544Z\",\n \"endTime\": \"2026-09-24T08:10:52.453305914Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8e8a0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-k9w78gnprtl2\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f599fa04-67e1-455d-a292-28baa109ae91\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f599fa04-67e1-455d-a292-28baa109ae91\",\n \"name\": \"f599fa04-67e1-455d-a292-28baa109ae91\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:46.285678544Z\",\n \"endTime\": \"2026-09-24T08:10:52.453305914Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9798c27f27d780103b1324a484958cc9bbe90fffa465297a059cec9bd20e84ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"500f59df8165aef3ffa121de8a9c035e4de2dc3301c2c3ddf3ea43873d0b709c","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f6a940\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-7489vnhpxmtv\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6d165f0-3e16-45c0-8ee1-64d82d0bdcdb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6d165f0-3e16-45c0-8ee1-64d82d0bdcdb\",\n \"name\": \"a6d165f0-3e16-45c0-8ee1-64d82d0bdcdb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:42.203685848Z\",\n \"endTime\": \"2026-09-24T08:09:42.350097216Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f6a940\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-7489vnhpxmtv\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6d165f0-3e16-45c0-8ee1-64d82d0bdcdb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6d165f0-3e16-45c0-8ee1-64d82d0bdcdb\",\n \"name\": \"a6d165f0-3e16-45c0-8ee1-64d82d0bdcdb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:42.203685848Z\",\n \"endTime\": \"2026-09-24T08:09:42.350097216Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"97e7004cade00a7ddf7ca3611a491715c275966e8b6df24beadd1276862e1d61","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"943aa3a834a0c2cefb204a1c45c66cc889d4122f1d10900e7220b69be2bceb26","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001108aa0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-vr2lq57sbxwl\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9866ffa9-5ecc-478f-9127-34fa7c51d525\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9866ffa9-5ecc-478f-9127-34fa7c51d525\",\n \"name\": \"9866ffa9-5ecc-478f-9127-34fa7c51d525\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:42:40.677418439Z\",\n \"endTime\": \"2026-09-24T08:11:42.538085729Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001108aa0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-vr2lq57sbxwl\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9866ffa9-5ecc-478f-9127-34fa7c51d525\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9866ffa9-5ecc-478f-9127-34fa7c51d525\",\n \"name\": \"9866ffa9-5ecc-478f-9127-34fa7c51d525\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:42:40.677418439Z\",\n \"endTime\": \"2026-09-24T08:11:42.538085729Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a58761098689b9125753464d095ea36f03d254e177c840e2eb673f1c41e55ece","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"alert","signature_id":"8a9c66eda94e766b0ef899f88f4ac1a07026419074e6b27f42b8d0218d54c88e","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T07:42:23Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-749558849-6dgkt\", region=\"westus3\", severity=\"warning\"\n Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-749558849-6dgkt has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"a795d0474316b2906472517f1b293d6f2e6c259229616ff1faed41452a94cd0a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"5940ace73d01ab95b0fb2644669c5a69b3f53304c116e4dc03542a7fde89df81","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000f4e930\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-mkjrz7qtzs5m\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-mkjrz7qtzs5m, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000f4e930\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-mkjrz7qtzs5m\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-mkjrz7qtzs5m, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-mkjrz7qtzs5m\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-mkjrz7qtzs5m, error: context deadline exceeded"}},{"row_id":"ad15e423236f40dcea290a8233f4bf0382a7e89d848f874b06e02d4484aab17b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"c445de35915f6d6808ebd93b49d8b192fef5dd052bad1e1a4994444e170535d0","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001534140\u003e: \n failed to create HCP cluster np-downgrade-7tpfr2: failed waiting for cluster=\"np-downgrade-7tpfr2\" in resourcegroup=\"rg-np-downgrade-7tpfr2-htscpw75fqv9\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82d495e5-dbce-4bfe-9466-1643b0df417b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82d495e5-dbce-4bfe-9466-1643b0df417b\",\n \"name\": \"82d495e5-dbce-4bfe-9466-1643b0df417b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:56.503596125Z\",\n \"endTime\": \"2026-09-24T08:11:02.701174589Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001534140\u003e: \n failed to create HCP cluster np-downgrade-7tpfr2: failed waiting for cluster=\"np-downgrade-7tpfr2\" in resourcegroup=\"rg-np-downgrade-7tpfr2-htscpw75fqv9\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82d495e5-dbce-4bfe-9466-1643b0df417b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/82d495e5-dbce-4bfe-9466-1643b0df417b\",\n \"name\": \"82d495e5-dbce-4bfe-9466-1643b0df417b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:56.503596125Z\",\n \"endTime\": \"2026-09-24T08:11:02.701174589Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b36db02bcadfe1e77893b6e797a9e809ff0d1f18d73783a67764a26bf2a5263f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"429c76b6d09d4b68e8144f42b4ace52233d91d9e8ee5711e33046c0d5d3cacb0","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b59000\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-vthts5p2dkf6\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d4d60ca-10b8-4b3d-b4c9-69db18d36300\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d4d60ca-10b8-4b3d-b4c9-69db18d36300\",\n \"name\": \"6d4d60ca-10b8-4b3d-b4c9-69db18d36300\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:48.958907495Z\",\n \"endTime\": \"2026-09-24T08:09:52.444767131Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b59000\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-vthts5p2dkf6\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d4d60ca-10b8-4b3d-b4c9-69db18d36300\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6d4d60ca-10b8-4b3d-b4c9-69db18d36300\",\n \"name\": \"6d4d60ca-10b8-4b3d-b4c9-69db18d36300\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:48.958907495Z\",\n \"endTime\": \"2026-09-24T08:09:52.444767131Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b702aadb3fb30bacfeff11c785b20cc5e452b0c2d7a134019b67152c2d43113f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"alert","signature_id":"6ffa9e2af62696b854cbaa4af470e9d2a538a83db7cca9ca00946bce22eef1d8","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T08:07:24Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8969088/providers/microsoft.monitor/accounts/services-j8969088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-7jtls\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-24T08:19:22Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8969088/providers/microsoft.monitor/accounts/services-j8969088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-7jtls\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-24T08:19:22Z\n Ended: 2026-09-24T08:37:22Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8969088/providers/microsoft.monitor/accounts/services-j8969088\", namespace=\"klusterlet-2t18bqdbena2h0p1tg4m0lfs0js71a72\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-7jtls\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t18bqdbena2h0p1tg4m0lfs0js71a72/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"bab478b1538b5cfd82c756577853235ae392598835bca3c07ad7c917d2801246","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"8fc3ce0aaf075ecc748035615ba580d85a206c822b6daabc625d94f5b3a03ac5","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-9wjkm6\" with version 4.20.38 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c07140\u003e: \n failed to create HCP cluster cluster-zstream-4-20-9wjkm6: failed waiting for cluster=\"cluster-zstream-4-20-9wjkm6\" in resourcegroup=\"rg-zstream-upgrade-4-20-ms4b72wr9z2w\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/50a9d6ed-b250-400e-b76f-acd47c11048b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/50a9d6ed-b250-400e-b76f-acd47c11048b\",\n \"name\": \"50a9d6ed-b250-400e-b76f-acd47c11048b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:59.932444459Z\",\n \"endTime\": \"2026-09-24T08:10:02.52795651Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-9wjkm6\" with version 4.20.38 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c07140\u003e: \n failed to create HCP cluster cluster-zstream-4-20-9wjkm6: failed waiting for cluster=\"cluster-zstream-4-20-9wjkm6\" in resourcegroup=\"rg-zstream-upgrade-4-20-ms4b72wr9z2w\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/50a9d6ed-b250-400e-b76f-acd47c11048b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/50a9d6ed-b250-400e-b76f-acd47c11048b\",\n \"name\": \"50a9d6ed-b250-400e-b76f-acd47c11048b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:59.932444459Z\",\n \"endTime\": \"2026-09-24T08:10:02.52795651Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"be77001ac88b9adab02fe3a680b458e00605419717b83c4a2a3ef4e27304a55c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"fd7b575d29435c0f905dacf7124fa2c4be1abd2c35d8c3bd9219b47920c3bc47","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-fq58btznb64f/cluster-candidate-4-22-wlgmvf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000950a60\u003e: \n failed to create HCP cluster cluster-candidate-4-22-wlgmvf: failed waiting for cluster=\"cluster-candidate-4-22-wlgmvf\" in resourcegroup=\"rg-candidate-4-22-fq58btznb64f\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec4a1266-5c44-4f1b-b4db-4746549d89a3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec4a1266-5c44-4f1b-b4db-4746549d89a3\",\n \"name\": \"ec4a1266-5c44-4f1b-b4db-4746549d89a3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:45:08.82425181Z\",\n \"endTime\": \"2026-09-24T08:14:12.55425933Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-fq58btznb64f/cluster-candidate-4-22-wlgmvf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000950a60\u003e: \n failed to create HCP cluster cluster-candidate-4-22-wlgmvf: failed waiting for cluster=\"cluster-candidate-4-22-wlgmvf\" in resourcegroup=\"rg-candidate-4-22-fq58btznb64f\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec4a1266-5c44-4f1b-b4db-4746549d89a3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ec4a1266-5c44-4f1b-b4db-4746549d89a3\",\n \"name\": \"ec4a1266-5c44-4f1b-b4db-4746549d89a3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:45:08.82425181Z\",\n \"endTime\": \"2026-09-24T08:14:12.55425933Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cacad1508900040c810464a75187c918cdc2819331999f8d45dc26a60d23eb8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"alert","signature_id":"db9539a235d0c98a40af1909968de9adb2c176e0ca9c6ae7df7436f7869eb782","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T08:19:22Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8969088/providers/microsoft.monitor/accounts/services-j8969088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-7jtls\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-24T08:23:25Z\n Ended: 2026-09-24T08:37:24Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8969088/providers/microsoft.monitor/accounts/services-j8969088\", namespace=\"klusterlet-2t18bqdbena2h0p1tg4m0lfs0js71a72\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-7jtls\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment klusterlet-2t18bqdbena2h0p1tg4m0lfs0js71a72/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-24T08:44:23Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8969088/providers/microsoft.monitor/accounts/services-j8969088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-7jtls\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired"}},{"row_id":"d42b7bf2acadb98a81c2355c6d522d0c039a981609b191a4a9c17a0292e52cde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"1c0d63e4358fa1d03a508395235eb4276e75a2a3b1c57828b7c15799aa97cac3","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001042820\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-865gq4n4j7qs\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/281a3b36-eef9-463a-9b25-42d8b8e21392\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/281a3b36-eef9-463a-9b25-42d8b8e21392\",\n \"name\": \"281a3b36-eef9-463a-9b25-42d8b8e21392\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:58.266564607Z\",\n \"endTime\": \"2026-09-24T08:10:03.037476406Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001042820\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-865gq4n4j7qs\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/281a3b36-eef9-463a-9b25-42d8b8e21392\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/281a3b36-eef9-463a-9b25-42d8b8e21392\",\n \"name\": \"281a3b36-eef9-463a-9b25-42d8b8e21392\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:58.266564607Z\",\n \"endTime\": \"2026-09-24T08:10:03.037476406Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d9f043365025d77de3927a294c4ca1bdb28f180f502364575755b88881b56266","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"6e4748bfac20ea730efd2ae4f326feca89e989b0d1171f412e6d519f9e1670e7","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d00880\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-f9xd6hpmvwx7\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/397f059a-151e-4d29-9587-6f24130b2671\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/397f059a-151e-4d29-9587-6f24130b2671\",\n \"name\": \"397f059a-151e-4d29-9587-6f24130b2671\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:05.679118342Z\",\n \"endTime\": \"2026-09-24T08:10:12.302829561Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d00880\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-f9xd6hpmvwx7\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/397f059a-151e-4d29-9587-6f24130b2671\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/397f059a-151e-4d29-9587-6f24130b2671\",\n \"name\": \"397f059a-151e-4d29-9587-6f24130b2671\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:05.679118342Z\",\n \"endTime\": \"2026-09-24T08:10:12.302829561Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"deae88097a0767857644c0d66f1163a6ff97ad89d3b2be2a309247ee90ced3b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"c92e73f72644876ddde62cf422ddb3b39fc8bfa8297fb43d32cbf01215c8b722","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd8500\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-2hhmscbkprxl\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b5624c4-af32-45f7-abd4-997b14ba3ff0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b5624c4-af32-45f7-abd4-997b14ba3ff0\",\n \"name\": \"6b5624c4-af32-45f7-abd4-997b14ba3ff0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:43:13.088665832Z\",\n \"endTime\": \"2026-09-24T08:12:22.246717537Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd8500\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-2hhmscbkprxl\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b5624c4-af32-45f7-abd4-997b14ba3ff0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b5624c4-af32-45f7-abd4-997b14ba3ff0\",\n \"name\": \"6b5624c4-af32-45f7-abd4-997b14ba3ff0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:43:13.088665832Z\",\n \"endTime\": \"2026-09-24T08:12:22.246717537Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e2e22e214a57ba47cc55ff7b8ca9050cebd7ce315dfad974b4f1aa1f38baf46a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"b5ccca774add3458c9bfae3e4c9d277eb98b1b593daf2a2bb7c71a1a8090c02f","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-fsqvbv with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000896da0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-fsqvbv: failed waiting for cluster=\"np-version-upgrade-cluster-fsqvbv\" in resourcegroup=\"rg-np-version-upgrade-fsqvbv-zcb48kjc4hbz\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b9dcfd00-26f9-4831-add2-9063aca7c365\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b9dcfd00-26f9-4831-add2-9063aca7c365\",\n \"name\": \"b9dcfd00-26f9-4831-add2-9063aca7c365\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:43:29.112056745Z\",\n \"endTime\": \"2026-09-24T08:12:32.442212915Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-fsqvbv with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000896da0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-fsqvbv: failed waiting for cluster=\"np-version-upgrade-cluster-fsqvbv\" in resourcegroup=\"rg-np-version-upgrade-fsqvbv-zcb48kjc4hbz\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b9dcfd00-26f9-4831-add2-9063aca7c365\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b9dcfd00-26f9-4831-add2-9063aca7c365\",\n \"name\": \"b9dcfd00-26f9-4831-add2-9063aca7c365\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:43:29.112056745Z\",\n \"endTime\": \"2026-09-24T08:12:32.442212915Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f2b2abba8bc117e6a3254073d48c970afd6c39cadc884645d3bd3681a98e4f7f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"1eff726a97351513b4ec821562ed516277791ca031fbd1e1aa1de9c857212fba","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bec5c0\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-6x5sfh8hdkmp\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e72784f9-ed82-4aaf-b2c5-bbc2ebf86dfc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e72784f9-ed82-4aaf-b2c5-bbc2ebf86dfc\",\n \"name\": \"e72784f9-ed82-4aaf-b2c5-bbc2ebf86dfc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:46.342831081Z\",\n \"endTime\": \"2026-09-24T08:10:52.289372657Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bec5c0\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-6x5sfh8hdkmp\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e72784f9-ed82-4aaf-b2c5-bbc2ebf86dfc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e72784f9-ed82-4aaf-b2c5-bbc2ebf86dfc\",\n \"name\": \"e72784f9-ed82-4aaf-b2c5-bbc2ebf86dfc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:41:46.342831081Z\",\n \"endTime\": \"2026-09-24T08:10:52.289372657Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f7716084b51453f74c3cb565ee14be84855448b8cc4f5125d6c257d7ffe6fe24","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"e538f2a0230f20027d8293b5a6c68382ea045c9e1651af9ed3dd5ac5b523af39","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a393c0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-79cbhfwcdzrd\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4d9d3b3e-4756-42b1-828c-2403d8fff2d6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4d9d3b3e-4756-42b1-828c-2403d8fff2d6\",\n \"name\": \"4d9d3b3e-4756-42b1-828c-2403d8fff2d6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:59.321420056Z\",\n \"endTime\": \"2026-09-24T08:10:02.462685071Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a393c0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-79cbhfwcdzrd\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4d9d3b3e-4756-42b1-828c-2403d8fff2d6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4d9d3b3e-4756-42b1-828c-2403d8fff2d6\",\n \"name\": \"4d9d3b3e-4756-42b1-828c-2403d8fff2d6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:59.321420056Z\",\n \"endTime\": \"2026-09-24T08:10:02.462685071Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f77a785f04337639a146e247dc3d410a79c71742c493ea9a2a9367ab1213145a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"bb3afbdbc42ca49133c243c7445c3bbf9eb564d2c828e805ba0ded344b330675","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001188640\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-xljgmjmsk6pr\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75063baf-064b-4133-bc96-b302135a5d38\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75063baf-064b-4133-bc96-b302135a5d38\",\n \"name\": \"75063baf-064b-4133-bc96-b302135a5d38\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:43:31.189652846Z\",\n \"endTime\": \"2026-09-24T08:12:32.306902555Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001188640\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-xljgmjmsk6pr\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75063baf-064b-4133-bc96-b302135a5d38\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75063baf-064b-4133-bc96-b302135a5d38\",\n \"name\": \"75063baf-064b-4133-bc96-b302135a5d38\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:43:31.189652846Z\",\n \"endTime\": \"2026-09-24T08:12:32.306902555Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fcd356d4fb2d1b6339bb7a7c1e8d29e7b9830876c087983171f085b37fad6fe5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","failed_at":"e2e","signature_id":"8a6e31d88bf2e3e5d322a70ec29117555c04f8de5f296cdc0c68246bbf060ddf","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-x4ll2j\" with version 5.0.0-rc.2 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00111a880\u003e: \n failed to create HCP cluster cluster-zstream-5-0-x4ll2j: failed waiting for cluster=\"cluster-zstream-5-0-x4ll2j\" in resourcegroup=\"rg-zstream-upgrade-5-0-vlkr8pbx4527\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e0de306d-0bf4-4437-aead-307f991f1a5d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e0de306d-0bf4-4437-aead-307f991f1a5d\",\n \"name\": \"e0de306d-0bf4-4437-aead-307f991f1a5d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:49.469137955Z\",\n \"endTime\": \"2026-09-24T08:09:52.326335951Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-x4ll2j\" with version 5.0.0-rc.2 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00111a880\u003e: \n failed to create HCP cluster cluster-zstream-5-0-x4ll2j: failed waiting for cluster=\"cluster-zstream-5-0-x4ll2j\" in resourcegroup=\"rg-zstream-upgrade-5-0-vlkr8pbx4527\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e0de306d-0bf4-4437-aead-307f991f1a5d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e0de306d-0bf4-4437-aead-307f991f1a5d\",\n \"name\": \"e0de306d-0bf4-4437-aead-307f991f1a5d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T07:40:49.469137955Z\",\n \"endTime\": \"2026-09-24T08:09:52.326335951Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":51,"failure_pattern_ids":["4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","c4e2bd93d0410f48ddfa390cc77604492607a58b2a4eaff204baa6c7bed3af07","cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":51,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103012742733500416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6915,"pr_state":"open","pr_sha":"e207b76e733cb65c677c16c7f38706c45baff4cd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T06:44:44Z","started_at":"2026-09-24T06:44:44Z","completed_at":"2026-09-24T09:03:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T09:13:58.823734257Z","timing_metadata_checked_at":"2026-09-24T09:13:58.823734257Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T09:13:58.826270949Z","region_metadata_checked_at":"2026-09-24T09:13:58.826270949Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"93efb24b4732a5a52259b785b282938b5b06ec9d0547b37feae1e15093c89176","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103012742733500416","occurred_at":"2026-09-24T06:44:44Z","failed_at":"e2e","signature_id":"c66a01613e5fe6442ebdb12c9ea63dd7c9f1f53d810adbdc9b51b498b061280c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-bxncxk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b16880\u003e: \n DNS for route host agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-bxncxk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b16880\u003e: \n DNS for route host agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}},{"row_id":"b45c7deb36de3f176db8d85fd7e59620ef939055dcacb49a7a2b1ac2a92bd6b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103012742733500416","occurred_at":"2026-09-24T06:44:44Z","failed_at":"e2e","signature_id":"03a9333f9dd2535bd3ce94a401581227e326295231899b72504aa1c180c57341","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fec560\u003e: \n DNS for route host agnhost-e2e-sample-app-f5rnl.apps.aro.v20260901.t5ah.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-f5rnl.apps.aro.v20260901.t5ah.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-f5rnl.apps.aro.v20260901.t5ah.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fec560\u003e: \n DNS for route host agnhost-e2e-sample-app-f5rnl.apps.aro.v20260901.t5ah.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-f5rnl.apps.aro.v20260901.t5ah.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-f5rnl.apps.aro.v20260901.t5ah.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-f5rnl.apps.aro.v20260901.t5ah.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-f5rnl.apps.aro.v20260901.t5ah.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-f5rnl.apps.aro.v20260901.t5ah.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7126/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103010871180529664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7126,"pr_state":"open","pr_sha":"165a6bee007299d0b02f2260490b1a19a1b99e7d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T06:37:18Z","started_at":"2026-09-24T06:37:18Z","completed_at":"2026-09-24T08:53:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T09:03:31.54576765Z","timing_metadata_checked_at":"2026-09-24T09:03:31.54576765Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T09:03:31.552516928Z","region_metadata_checked_at":"2026-09-24T09:03:31.552516928Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"3b4253d9f04113f653be2e51328912e2ca14ed3a454f1d475dab4fcbdd40de78","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7126/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103010871180529664","occurred_at":"2026-09-24T06:37:18Z","failed_at":"e2e","signature_id":"3f39acc6e073ba27c21beb42a91e286c5d52972e441b176f4734975709089b1e","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.006s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc002b4cc50\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.006s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc002b4cc50\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}},{"row_id":"50dd353d3a0ccb9e25e5951fc7bd2567add5cc2f013a68ff38071e22291648bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7126/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103010871180529664","occurred_at":"2026-09-24T06:37:18Z","failed_at":"alert","signature_id":"a0b9e359a9ec67136305ac5041742a0b1b9f95678b9b1c144a344ebc7e1e7a01","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T08:09:57Z\n Ended: 2026-09-24T08:15:50Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0529664-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0529664-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"a33b9337691639b161990de2d04cc58a695588940a5722a9cdbe0e157824b1f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7126/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103010871180529664","occurred_at":"2026-09-24T06:37:18Z","failed_at":"alert","signature_id":"10f8755438a83f36559eb45ee70fb70663d5582f7bec31700fca4ba57353d6f8","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T08:09:49Z\n Ended: 2026-09-24T08:23:47Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j0529664-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-894df5fcb-zvw9n\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-894df5fcb-zvw9n (namespace kube-applier) on cluster ci01-j0529664-mgmt-1 has averaged 132.1% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103006860184391680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"closed","pr_sha":"87e40a07142a1457ed3f83374cc5d085c87d8f29","final_merged_sha":"e772682c6ada7bf3fd5674946d72fe25f79d834b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T06:21:22Z","started_at":"2026-09-24T06:21:22Z","completed_at":"2026-09-24T08:24:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T08:31:54.539434018Z","timing_metadata_checked_at":"2026-09-24T08:31:54.539434018Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T08:31:54.54216781Z","region_metadata_checked_at":"2026-09-24T08:31:54.54216781Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"58193e5608df5318e086c38ce8c657535bb92eaeab36e0a84437a1b39c8ffaa5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103006860184391680","occurred_at":"2026-09-24T06:21:22Z","failed_at":"e2e","signature_id":"613a9b7adaccf1b621c4d00e5e9f881ac489c8f71ae2c82dddfa7345990fdc47","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:273]: Failed after 0.060s.\npod on cordoned node x6j5o0j5b0f4m6v-cordon-np-xzf6f-b82cp should remain Pending\nThe function passed to Consistently failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:269 with:\npod targeting cordoned node x6j5o0j5b0f4m6v-cordon-np-xzf6f-b82cp should not be assigned a node\nExpected\n \u003cstring\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:273]: Failed after 0.060s.\npod on cordoned node x6j5o0j5b0f4m6v-cordon-np-xzf6f-b82cp should remain Pending\nThe function passed to Consistently failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:269 with:\npod targeting cordoned node x6j5o0j5b0f4m6v-cordon-np-xzf6f-b82cp should not be assigned a node\nExpected\n \u003cstring\u003e: ...\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36ebd23c937dc29d944a27853b2ad54818920906b2bd4877347b2a0825abe2fc","failure_pattern":"pod targeting cordoned node \u003cnode\u003e should not be assigned a node","search_query":"pod targeting cordoned node x6j5o0j5b0f4m6v-cordon-np-xzf6f-b82cp should not be assigned a node"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36ebd23c937dc29d944a27853b2ad54818920906b2bd4877347b2a0825abe2fc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103001507350712320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7105,"pr_state":"open","pr_sha":"ea520febfe3d628640b68cfe02f923dc6cf19b80","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T06:00:05Z","started_at":"2026-09-24T06:00:05Z","completed_at":"2026-09-24T08:05:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T08:10:40.523106668Z","timing_metadata_checked_at":"2026-09-24T08:10:40.523106668Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T08:10:40.529463548Z","region_metadata_checked_at":"2026-09-24T08:10:40.529463548Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"607dc2161c3da33a5f3b007ca3af2fec04d0a028f41283bc1fdda56fb83746b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103001507350712320","occurred_at":"2026-09-24T06:00:05Z","failed_at":"alert","signature_id":"6fd93723b17ab024ac620ec98b9e4488b425d7110afc72440506d1c61f3eab4d","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T07:16:34Z\n Ended: 2026-09-24T07:28:33Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j0712320-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-777c6ff944-dn4nc\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-777c6ff944-dn4nc (namespace kube-applier) on cluster ci01-j0712320-mgmt-1 has averaged 126.9% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7125/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102990771102683136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7125,"pr_state":"open","pr_sha":"76c71e426a88f7bfbf2c34cc6e3a1c046ec4babd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-24T05:17:17Z","started_at":"2026-09-24T05:17:17Z","completed_at":"2026-09-24T07:23:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T07:27:30.679414187Z","timing_metadata_checked_at":"2026-09-24T07:27:30.679414187Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T07:27:30.682171778Z","region_metadata_checked_at":"2026-09-24T07:27:30.682171778Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102989394158817280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5562,"pr_state":"closed","pr_sha":"18cce3707ed7df30b1a12d9779a8035dbdc24710","final_merged_sha":"18cce3707ed7df30b1a12d9779a8035dbdc24710","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T05:11:57Z","started_at":"2026-09-24T05:11:57Z","completed_at":"2026-09-24T07:16:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T07:27:30.527379587Z","timing_metadata_checked_at":"2026-09-24T07:27:30.527379587Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T07:27:30.530903076Z","region_metadata_checked_at":"2026-09-24T07:27:30.530903076Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"1e842ed3afa95b44a0e48dadb20568a361a10c0cdbac64f09c024a09d7e54f68","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102989394158817280","occurred_at":"2026-09-24T05:11:57Z","failed_at":"alert","signature_id":"e02ac96b476af77a18c8814f0b86746bd97b987ea683b9a1aa418df78ed20adc","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T06:14:05Z\n Ended: 2026-09-24T06:39:07Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j8817280-svc\", component=\"frontend\", method=\"post\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/requestadmincredential\", severity=\"warning\"\n Description: More than 5% of frontend requests for post /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/requestadmincredential exceeded 1 second over the past 30 minutes (cluster ci01-j8817280-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"231cd8aeba7a9a006c525f4a917c0f1e52757cefb2bfe83297c3beeb9f520eca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102989394158817280","occurred_at":"2026-09-24T05:11:57Z","failed_at":"alert","signature_id":"bd015bfe1b7ee0302c28b53124982a3fc176057803cf96dcb50518f45695f0ed","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T06:25:06Z\n Ended: 2026-09-24T06:31:05Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8817280-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8817280-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"695efc155ebe9f1886df57efec277265ec2e5a9c5c85de3907ffc5f4a31f470f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102989394158817280","occurred_at":"2026-09-24T05:11:57Z","failed_at":"alert","signature_id":"5417773a0fcbc7245cd61bb18c7acb92d76ae842a3f206a9a2b3bef948abce1f","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T06:55:31Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j8817280-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-pk287\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-pk287 (namespace arobit) on cluster ci01-j8817280-svc has averaged 124.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102977046442414080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7115,"pr_state":"closed","pr_sha":"7b12aa355095c6b8e3e4cff4ccd0e36e501553b7","final_merged_sha":"7b12aa355095c6b8e3e4cff4ccd0e36e501553b7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-24T04:22:53Z","started_at":"2026-09-24T04:22:53Z","completed_at":"2026-09-24T06:32:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T06:36:26.177244483Z","timing_metadata_checked_at":"2026-09-24T06:36:26.177244483Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T06:36:26.180550274Z","region_metadata_checked_at":"2026-09-24T06:36:26.180550274Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102965797063233536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6917,"pr_state":"closed","pr_sha":"393527983bbf3810fedd8ca66cc790d3d1526978","final_merged_sha":"393527983bbf3810fedd8ca66cc790d3d1526978","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T03:38:11Z","started_at":"2026-09-24T03:38:11Z","completed_at":"2026-09-24T05:34:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T05:44:28.017445224Z","timing_metadata_checked_at":"2026-09-24T05:44:28.017445224Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T05:44:28.021044312Z","region_metadata_checked_at":"2026-09-24T05:44:28.021044312Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"88aad6f30d59eddcbfd662ae70536653dc00e640d2a3d5f2b161ea9d01f7d118","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102965797063233536","occurred_at":"2026-09-24T03:38:11Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T04:25:52Z\n Ended: 2026-09-24T04:43:55Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3233536-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102961799233015808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7112,"pr_state":"open","pr_sha":"395e45d0bbde804a9c22a549e978e9371ff65d91","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T03:22:14Z","started_at":"2026-09-24T03:22:14Z","completed_at":"2026-09-24T05:23:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T05:34:04.01445098Z","timing_metadata_checked_at":"2026-09-24T05:34:04.01445098Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T05:34:04.017779269Z","region_metadata_checked_at":"2026-09-24T05:34:04.017779269Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"f5002ad5993676bdbc34464b5ea4082827e4ae438baa03b3b79eea11c26e71c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102961799233015808","occurred_at":"2026-09-24T03:22:14Z","failed_at":"alert","signature_id":"ce1c532ec2e46cd04d99a6539b8974a73f6fa6a9c1f96c6db3d23304ad9e593b","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T05:10:05Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j3015808-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-5dkbm\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-5dkbm (namespace arobit) on cluster ci01-j3015808-svc has averaged 125.9% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102949549692686336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7124,"pr_state":"open","pr_sha":"63d123b25ed907af9b77df97959feb931ad91b70","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-24T02:33:38Z","started_at":"2026-09-24T02:33:38Z","completed_at":"2026-09-24T04:49:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T05:02:41.685881035Z","timing_metadata_checked_at":"2026-09-24T05:02:41.685881035Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T05:02:41.689333424Z","region_metadata_checked_at":"2026-09-24T05:02:41.689333424Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6733/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102938362976407552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6733,"pr_state":"closed","pr_sha":"7df09e82565752899c9669782b8ac3ab38a07b3f","final_merged_sha":"7df09e82565752899c9669782b8ac3ab38a07b3f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-24T01:49:11Z","started_at":"2026-09-24T01:49:11Z","completed_at":"2026-09-24T03:54:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T03:58:55.062020774Z","timing_metadata_checked_at":"2026-09-24T03:58:55.062020774Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T03:58:55.066186764Z","region_metadata_checked_at":"2026-09-24T03:58:55.066186764Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102931134991044608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6226,"pr_state":"closed","pr_sha":"9b50264285a60d3666e3df261a54ae027b4564e9","final_merged_sha":"9b50264285a60d3666e3df261a54ae027b4564e9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T01:20:27Z","started_at":"2026-09-24T01:20:27Z","completed_at":"2026-09-24T03:33:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T03:38:30.649337037Z","timing_metadata_checked_at":"2026-09-24T03:38:30.649337037Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T03:38:30.653128726Z","region_metadata_checked_at":"2026-09-24T03:38:30.653128726Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"7ea5c9f4a4117cffc578df58a21346dfec69d4b46db28c8b8f9a80631be7689d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102931134991044608","occurred_at":"2026-09-24T01:20:27Z","failed_at":"alert","signature_id":"5d2d7a79a877fd36a7ce50a19b88b5be576cc7d6481c13905922189eaaa22c37","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T02:24:07Z\n Ended: 2026-09-24T02:47:09Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1044608-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1044608-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7090/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102921738877669376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7090,"pr_state":"closed","pr_sha":"6747bd2682b02d1bbe260d5e69aab3a33431b9ad","final_merged_sha":"6747bd2682b02d1bbe260d5e69aab3a33431b9ad","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T00:43:07Z","started_at":"2026-09-24T00:43:07Z","completed_at":"2026-09-24T02:58:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T03:07:42.787870594Z","timing_metadata_checked_at":"2026-09-24T03:07:42.787870594Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T03:07:42.790477586Z","region_metadata_checked_at":"2026-09-24T03:07:42.790477586Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"b5a1ba00c3aa891a16b885c2fbe4983d5a68420410220cb6a39c978a9f28d24c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7090/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102921738877669376","occurred_at":"2026-09-24T00:43:07Z","failed_at":"alert","signature_id":"3b853c2ad740c6b27854bb7b28c66560f90ec20a918beecff56ae64e308ded54","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T01:27:10Z\n Ended: 2026-09-24T01:40:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7669376-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prom-agent-prometheus-1\", region=\"centralus\", severity=\"warning\"\n Description: Pod prometheus/prom-agent-prometheus-1 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102921426674651136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7115,"pr_state":"closed","pr_sha":"7b12aa355095c6b8e3e4cff4ccd0e36e501553b7","final_merged_sha":"7b12aa355095c6b8e3e4cff4ccd0e36e501553b7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T00:41:53Z","started_at":"2026-09-24T00:41:53Z","completed_at":"2026-09-24T02:49:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T02:57:42.85623668Z","timing_metadata_checked_at":"2026-09-24T02:57:42.85623668Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T02:57:42.858932471Z","region_metadata_checked_at":"2026-09-24T02:57:42.858932471Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"31a9abfeed57e10f3ee01b65e83cc64bb44dd4825a034cdc1a8e96831e000a6c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102921426674651136","occurred_at":"2026-09-24T00:41:53Z","failed_at":"alert","signature_id":"3797bbedc99fc8f57829a65247ea354a468bfe71a528c5bc92588f0d1aa0858f","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T02:35:27Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j4651136-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-hlh84\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-hlh84 (namespace arobit) on cluster ci01-j4651136-svc has averaged 125.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"40a44e9077fb6ab88a685bbf77f64ae582859f5ecaba20fd539e2c514907f652","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102921426674651136","occurred_at":"2026-09-24T00:41:53Z","failed_at":"alert","signature_id":"c9c5fd9f1bd379784e5f4e5bbdd029af5ef5b392fcb1f2d33857ec4a703e485d","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T02:02:42Z\n Ended: 2026-09-24T02:17:42Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4651136-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/externalauths/{externalauthname}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/externalauths/{externalauthname} has exceeded 250ms over the past 30 minutes (cluster ci01-j4651136-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"8e6c0017249d72543c70670c8aeaef88bba06892bfd7cbcc39d7da50e6abc174","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102921426674651136","occurred_at":"2026-09-24T00:41:53Z","failed_at":"alert","signature_id":"7715e65904649de36554c3fa389bc722801fea292dc922d83d7ddf72b37313e4","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T02:04:43Z\n Ended: 2026-09-24T02:17:41Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j4651136-svc\", component=\"frontend\", method=\"delete\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: More than 5% of frontend requests for delete /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j4651136-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7108/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102921419087155200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7108,"pr_state":"closed","pr_sha":"12386789e94c4b6ae1c0283ae43f012c89dd3125","final_merged_sha":"12386789e94c4b6ae1c0283ae43f012c89dd3125","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-24T00:41:51Z","started_at":"2026-09-24T00:41:51Z","completed_at":"2026-09-24T02:58:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T03:07:42.920029198Z","timing_metadata_checked_at":"2026-09-24T03:07:42.920029198Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T03:07:42.92279469Z","region_metadata_checked_at":"2026-09-24T03:07:42.92279469Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102919814468079616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6917,"pr_state":"closed","pr_sha":"393527983bbf3810fedd8ca66cc790d3d1526978","final_merged_sha":"393527983bbf3810fedd8ca66cc790d3d1526978","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T00:35:28Z","started_at":"2026-09-24T00:35:28Z","completed_at":"2026-09-24T02:42:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T02:57:42.621541729Z","timing_metadata_checked_at":"2026-09-24T02:57:42.621541729Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T02:57:42.624579119Z","region_metadata_checked_at":"2026-09-24T02:57:42.624579119Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"022df531c7044df0dc0a57287df8a11de8efe4560e7fb1d042cfd16b363e26c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102919814468079616","occurred_at":"2026-09-24T00:35:28Z","failed_at":"alert","signature_id":"eb34a8b64fb9114676f2191bea2c8d39236f5170336f8b7d13fb0795cc5edc83","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 32 time(s) (16 unknown, 16 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T01:26:59Z\n Ended: 2026-09-24T01:41:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t12nvqr38rk88v2oib5stvkqbcsin5h-r3r9m3h6q3b6v7a\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t12nvqr38rk88v2oib5stvkqbcsin5h-r3r9m3h6q3b6v7a/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-24T01:26:59Z\n Ended: 2026-09-24T01:42:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t12nvj5ubvth7im4d53igd84r482cjj-n2f1o7y4s2y9x7u\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t12nvj5ubvth7im4d53igd84r482cjj-n2f1o7y4s2y9x7u/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-24T01:26:59Z\n Ended: 2026-09-24T01:42:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t12o9jo7duvi4mjl85qtghmc0bu27i9-j6r7x7q1c9p4g7n\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t12o9jo7duvi4mjl85qtghmc0bu27i9-j6r7x7q1c9p4g7n/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-24T01:26:59Z\n Ended: 2026-09-24T01:42:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t12o0vs2j1bsj8jv3dba43qbnunnl1i-cilium-cl\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t12o0vs2j1bsj8jv3dba43qbnunnl1i-cilium-cl/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-24T01:26:59Z\n Ended: 2026-09-24T01:42:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t12oj8duo80lsd7fs443vtl4r4h0lgc-l0s9f8e4u7h0d7x\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t12oj8duo80lsd7fs443vtl4r4h0lgc-l0s9f8e4u7h0d7x/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-24T01:27:59Z\n Ended: 2026-09-24T01:42:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t12o7sttviaemub24n9cm5ag1dhpjn3-y5b7h7t6x0t6l3h\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t12o7sttviaemub24n9cm5ag1dhpjn3-y5b7h7t6x0t6l3h/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-24T01:27:59Z\n Ended: 2026-09-24T01:42:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t12ogvi1mc8j23fcmcidu1snj4udrjj-f4a9l6j0e1y1n5s\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t12ogvi1mc8j23fcmcidu1snj4udrjj-f4a9l6j0e1y1n5s/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-24T01:27:59Z\n Ended: 2026-09-24T01:42:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t12oie0obpjdbg2ir489o2m0hhqia63-y0z4j0l8g6d7i9c\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t12oie0obpjdbg2ir489o2m0hhqia63-y0z4j0l8g6d7i9c/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-24T01:28:00Z\n Ended: 2026-09-24T01:42:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t12o1tegps12avbee0567i1pg5n9aqp-z2z9h8s1b9q1t2n\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t12o1tegps12avbee0567i1pg5n9aqp-z2z9h8s1b9q1t2n/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-24T01:28:59Z\n Ended: 2026-09-24T01:42:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t12o3npaeb4jqp5q1ofrtc1092aoar1-c2g1v2c1l2t9t3r\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t12o3npaeb4jqp5q1ofrtc1092aoar1-c2g1v2c1l2t9t3r/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-24T01:28:59Z\n Ended: 2026-09-24T01:42:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t12oitpqsdnrnuh5pvf6ef0d0kumthn-k4q8w3o6t9k1j1t\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t12oitpqsdnrnuh5pvf6ef0d0kumthn-k4q8w3o6t9k1j1t/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-24T01:28:59Z\n Ended: 2026-09-24T01:42:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t12p5t7p0tpbrkhhf2dkfhhal6mv6p2-n2m3j4r2d5i3z1y\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t12p5t7p0tpbrkhhf2dkfhhal6mv6p2-n2m3j4r2d5i3z1y/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-24T01:29:58Z\n Ended: 2026-09-24T01:42:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t12nu9v6kjftfmhqg85gvvpie2uguej-o0b2r4o1s5g4u5f\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t12nu9v6kjftfmhqg85gvvpie2uguej-o0b2r4o1s5g4u5f/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-24T01:29:58Z\n Ended: 2026-09-24T01:42:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t12onoeddbqgl3ej67k3nqf0cik8cqe-u5k3g3h0e6g6t4p\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t12onoeddbqgl3ej67k3nqf0cik8cqe-u5k3g3h0e6g6t4p/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-09-24T01:29:58Z\n Ended: 2026-09-24T01:42:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t12ovpa0h0i72vs6ph53efpthtkuuvh-agg-cluster\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t12ovpa0h0i72vs6ph53efpthtkuuvh-agg-cluster/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-09-24T01:29:59Z\n Ended: 2026-09-24T01:42:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t12o0s164atnq3e0on0si79krsq0ffu-v1f8i4d8m8e3k7w\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t12o0s164atnq3e0on0si79krsq0ffu-v1f8i4d8m8e3k7w/etcd-1 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"9f8dedf5eb286923ddcd5ff800f1b71ecc961e3f08458930533981d3898998f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102919814468079616","occurred_at":"2026-09-24T00:35:28Z","failed_at":"alert","signature_id":"98313410c810095dec26c72ec7f4b43b7a09e0afa8d35824ba5fc5fec76fe165","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T01:36:59Z\n Ended: 2026-09-24T01:43:01Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8079616/providers/microsoft.monitor/accounts/hcps-j8079616\", namespace=\"ocm-arohcpci01-2t12nvqr38rk88v2oib5stvkqbcsin5h-r3r9m3h6q3b6v7a\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-q4z8l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t12nvqr38rk88v2oib5stvkqbcsin5h-r3r9m3h6q3b6v7a/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-24T01:36:59Z\n Ended: 2026-09-24T01:43:01Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8079616/providers/microsoft.monitor/accounts/hcps-j8079616\", namespace=\"ocm-arohcpci01-2t12o9jo7duvi4mjl85qtghmc0bu27i9-j6r7x7q1c9p4g7n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-q4z8l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t12o9jo7duvi4mjl85qtghmc0bu27i9-j6r7x7q1c9p4g7n/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-24T01:37:00Z\n Ended: 2026-09-24T01:43:01Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8079616/providers/microsoft.monitor/accounts/hcps-j8079616\", namespace=\"ocm-arohcpci01-2t12nvj5ubvth7im4d53igd84r482cjj-n2f1o7y4s2y9x7u\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-q4z8l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t12nvj5ubvth7im4d53igd84r482cjj-n2f1o7y4s2y9x7u/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-24T01:37:00Z\n Ended: 2026-09-24T01:43:01Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8079616/providers/microsoft.monitor/accounts/hcps-j8079616\", namespace=\"ocm-arohcpci01-2t12o9jo7duvi4mjl85qtghmc0bu27i9-j6r7x7q1c9p4g7n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-q4z8l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t12o9jo7duvi4mjl85qtghmc0bu27i9-j6r7x7q1c9p4g7n/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102916135383994368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6821,"pr_state":"closed","pr_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","final_merged_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T00:20:51Z","started_at":"2026-09-24T00:20:51Z","completed_at":"2026-09-24T01:28:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T01:33:52.160974242Z","timing_metadata_checked_at":"2026-09-24T01:33:52.160974242Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T01:33:52.163882933Z","region_metadata_checked_at":"2026-09-24T01:33:52.163882933Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"56cc4af98b87c4d981ffb2d1e551e5defe366151cd5035d0e8176d6f67d68d1f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102916135383994368","occurred_at":"2026-09-24T00:20:51Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102916121228218368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7054,"pr_state":"closed","pr_sha":"f0e1fa3e33ffc30bc4eeec511d85f73874d58b9b","final_merged_sha":"7553575593411fb5f3c1d57088aa752a234c76d1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T00:20:48Z","started_at":"2026-09-24T00:20:48Z","completed_at":"2026-09-24T02:39:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T02:46:53.381851888Z","timing_metadata_checked_at":"2026-09-24T02:46:53.381851888Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T02:46:53.385508177Z","region_metadata_checked_at":"2026-09-24T02:46:53.385508177Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"9efedac9ec34b4377fa48c9df31a9f663b5efd681e4c7f952e4f15510bc4c3f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102916121228218368","occurred_at":"2026-09-24T00:20:48Z","failed_at":"alert","signature_id":"87d3d8dba8e6534afffc2d80f759bd53c5872f8a0e40478e7a7531c5b2e7dae9","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T01:44:51Z\n Ended: 2026-09-24T02:00:48Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j8218368-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-844b448d88-cdbrn\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-844b448d88-cdbrn (namespace kube-applier) on cluster ci01-j8218368-mgmt-1 has averaged 131.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102915118240108544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7119,"pr_state":"open","pr_sha":"06b9f9889389f9b6bf8ca6bd99239ae9cb9ee430","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T00:16:20Z","started_at":"2026-09-24T00:16:20Z","completed_at":"2026-09-24T02:33:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T02:46:53.632321477Z","timing_metadata_checked_at":"2026-09-24T02:46:53.632321477Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T02:46:53.635649067Z","region_metadata_checked_at":"2026-09-24T02:46:53.635649067Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"0a014eb84ec06a19e3a74d1ee5ba508e66136262bf8134bc0ee185d92fea4fd7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102915118240108544","occurred_at":"2026-09-24T00:16:20Z","failed_at":"e2e","signature_id":"e6dd7dea20380c21745a379f6c7147226e56bf52ffe41815559c0e6f0dbfefbd","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d99500\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-bfzfjm\" in resourcegroup=\"rg-np-version-upgrade-bfzfjm-pnjhht5f5zwq\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/49267939-19c8-4acf-921e-89eba56b2728\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/49267939-19c8-4acf-921e-89eba56b2728\",\n \"name\": \"49267939-19c8-4acf-921e-89eba56b2728\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T01:19:02.701552742Z\",\n \"endTime\": \"2026-09-24T01:19:42.41444708Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d99500\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-bfzfjm\" in resourcegroup=\"rg-np-version-upgrade-bfzfjm-pnjhht5f5zwq\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/49267939-19c8-4acf-921e-89eba56b2728\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/49267939-19c8-4acf-921e-89eba56b2728\",\n \"name\": \"49267939-19c8-4acf-921e-89eba56b2728\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T01:19:02.701552742Z\",\n \"endTime\": \"2026-09-24T01:19:42.41444708Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8e6c1c973557413787b2a3d5f92927f9a1933a2ef6e823e2772adc38897be9b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102915118240108544","occurred_at":"2026-09-24T00:16:20Z","failed_at":"e2e","signature_id":"12c0bbf57d474e0cd6cf2451ff2cf882646f3b861a1cce2aae346ddfcc9bb6c4","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-lqvvv444d6v8\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3af991c4-642e-4bf3-b698-91174beba451\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3af991c4-642e-4bf3-b698-91174beba451\",\n \"name\": \"3af991c4-642e-4bf3-b698-91174beba451\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T01:19:13.956261131Z\",\n \"endTime\": \"2026-09-24T01:19:51.914301535Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n}\n--------------------------------------------------------------------------------\n failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-lqvvv444d6v8\" to finish creating: context canceled]\nExpected\n \u003c[]error | len:2, cap:2\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-lqvvv444d6v8\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3af991c4-642e-4bf3-b698-91174beba451\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3af991c4-642e-4bf3-b698-91174beba451\",\n \"name\": \"3af991c4-642e-4bf3-b698-91174beba451\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T01:19:13.956261131Z\",\n \"endTime\": \"2026-09-24T01:19:51.914301535Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n}\n--------------------------------------------------------------------------------\n failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-lqvvv444d6v8\" to finish creating: context canceled]\nExpected\n \u003c[]error | len:2, cap:2\u003e: ...\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e86119606257db722a985501dec6db3353e9294ec584e08a2cf350342957ce78","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context canceled]","search_query":"failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-lqvvv444d6v8\" to finish creating: context canceled]"}},{"row_id":"a3f8c1e96ad4735203e9c99ac16dd0d1f9c5f796a355d09e7562fcc3297c8608","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102915118240108544","occurred_at":"2026-09-24T00:16:20Z","failed_at":"alert","signature_id":"30e0e523f1fde040099a61098a3eec7b89130d7155eceabe4360309fa9b4a812","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T01:45:46Z\n Ended: 2026-09-24T01:51:43Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0108544-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0108544-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"b425df197035fac43387ad4d688371334b277a9c2c1094ce5887ee768cc98614","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102915118240108544","occurred_at":"2026-09-24T00:16:20Z","failed_at":"e2e","signature_id":"88da56dc9cbaf5632d307314826460b6bc87c0605368c5a80838d8aa0648f7a2","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00102af00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6g69q88jrgvp\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/03af9a97-911f-44d7-b300-118b68afbfc6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/03af9a97-911f-44d7-b300-118b68afbfc6\",\n \"name\": \"03af9a97-911f-44d7-b300-118b68afbfc6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T01:19:16.891877526Z\",\n \"endTime\": \"2026-09-24T01:19:52.061288026Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00102af00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6g69q88jrgvp\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/03af9a97-911f-44d7-b300-118b68afbfc6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/03af9a97-911f-44d7-b300-118b68afbfc6\",\n \"name\": \"03af9a97-911f-44d7-b300-118b68afbfc6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T01:19:16.891877526Z\",\n \"endTime\": \"2026-09-24T01:19:52.061288026Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b70427c4adec017d81bebe9c78905663dc052c48df928361518b7270197c9aa8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102915118240108544","occurred_at":"2026-09-24T00:16:20Z","failed_at":"e2e","signature_id":"f9b1fc76540f4c53a5a55de6e48304b8e30d158e9f8f3534a80d2ae685ba6b26","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006d7640\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-r7s2j5xpgxj7\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2c460257-6fa9-47d3-ae45-d13bd8450002\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2c460257-6fa9-47d3-ae45-d13bd8450002\",\n \"name\": \"2c460257-6fa9-47d3-ae45-d13bd8450002\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T01:20:17.897504202Z\",\n \"endTime\": \"2026-09-24T01:20:42.081671064Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006d7640\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-r7s2j5xpgxj7\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2c460257-6fa9-47d3-ae45-d13bd8450002\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2c460257-6fa9-47d3-ae45-d13bd8450002\",\n \"name\": \"2c460257-6fa9-47d3-ae45-d13bd8450002\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T01:20:17.897504202Z\",\n \"endTime\": \"2026-09-24T01:20:42.081671064Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bdde2c9857a17bfa1209a3634cf34f322dbd9e7050ce6cfcbd26e46b63f1f711","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102915118240108544","occurred_at":"2026-09-24T00:16:20Z","failed_at":"e2e","signature_id":"8bf81576a6cc6f8137c7015977c9d4e64ecd2342fb685d20f3e351e64bbf3b0b","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d49b40\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-mq6zsk\" in resourcegroup=\"rg-np-noedge-mq6zsk-fghfbkj4p4f7\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/47cd64e6-999c-4582-86db-82685b5779de\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/47cd64e6-999c-4582-86db-82685b5779de\",\n \"name\": \"47cd64e6-999c-4582-86db-82685b5779de\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T01:19:02.94932145Z\",\n \"endTime\": \"2026-09-24T01:19:42.326658933Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d49b40\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-mq6zsk\" in resourcegroup=\"rg-np-noedge-mq6zsk-fghfbkj4p4f7\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/47cd64e6-999c-4582-86db-82685b5779de\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/47cd64e6-999c-4582-86db-82685b5779de\",\n \"name\": \"47cd64e6-999c-4582-86db-82685b5779de\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T01:19:02.94932145Z\",\n \"endTime\": \"2026-09-24T01:19:42.326658933Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","e86119606257db722a985501dec6db3353e9294ec584e08a2cf350342957ce78"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102914094729269248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7123,"pr_state":"closed","pr_sha":"3eb10bd2f1feb2c6d62a69754d5b84aa95d9f23b","final_merged_sha":"c07c99eb845db32bdffea8c5c152b60fa095f319","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T00:12:45Z","started_at":"2026-09-24T00:12:45Z","completed_at":"2026-09-24T02:35:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T02:46:53.74383496Z","timing_metadata_checked_at":"2026-09-24T02:46:53.74383496Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T02:46:53.746488353Z","region_metadata_checked_at":"2026-09-24T02:46:53.746488353Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"faa5be1ab367eb1a85e84750a99bc9e8f678df2c566aacdfb31bfdb551a77b32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102914094729269248","occurred_at":"2026-09-24T00:12:45Z","failed_at":"alert","signature_id":"79341176e7e5e053b430da83424bffbe33d8cbf6966dd23ee9a8f0cd6624d271","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T01:47:00Z\n Ended: 2026-09-24T02:00:56Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j9269248-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-5f4587fffd-wtn5k\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container kube-applier in pod kube-applier-5f4587fffd-wtn5k (namespace kube-applier) on cluster ci01-j9269248-mgmt-1 has averaged 128.7% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102913782756937728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7112,"pr_state":"open","pr_sha":"395e45d0bbde804a9c22a549e978e9371ff65d91","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T00:11:30Z","started_at":"2026-09-24T00:11:30Z","completed_at":"2026-09-24T02:38:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T02:46:53.495663065Z","timing_metadata_checked_at":"2026-09-24T02:46:53.495663065Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T02:46:53.498746456Z","region_metadata_checked_at":"2026-09-24T02:46:53.498746456Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"295bf89b6d5caa90d1a9b3d13139a7d48e4e4946fb26ad753996b376f83ddf39","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102913782756937728","occurred_at":"2026-09-24T00:11:30Z","failed_at":"e2e","signature_id":"9fce1ec92cfb95fb8f8175f8912efb491fd5065485198557528e2813ccb986fe","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:216]: Timed out after 600.001s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:214 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host\nUnexpected error:\n \u003c*url.Error | 0xc001758f60\u003e: \n Get \"https://console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:216]: Timed out after 600.001s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:214 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host\nUnexpected error:\n \u003c*url.Error | 0xc001758f60\u003e: \n Get \"https://console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d47ff99fd1bcd56f3dd1a2d45fb345dc45ebaade86da8c2155a6cfa662ad0280","failure_pattern":"public ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"public ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host"}},{"row_id":"52022e2fa2fce65fb46bf0ce3227cbd4c7fddf1f9182b2369c057ec80f7d93b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102913782756937728","occurred_at":"2026-09-24T00:11:30Z","failed_at":"alert","signature_id":"c5508e299b7fba05140b4f25dbef14c99e81680c4220a923738b6a2265553a53","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-24T01:15:55Z\n Ended: 2026-09-24T01:48:49Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6937728-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6937728-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"fea83dc1afe30e0091c7fe5058d8e0c7acc04f23d3ac178381f1c4c3dafce745","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102913782756937728","occurred_at":"2026-09-24T00:11:30Z","failed_at":"alert","signature_id":"739f026725e19a96d09fe6b2030f5896718a89453fad23e329ee1552ca883a94","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T02:09:06Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j6937728-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-br8g5\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-br8g5 (namespace arobit) on cluster ci01-j6937728-svc has averaged 126.4% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["d47ff99fd1bcd56f3dd1a2d45fb345dc45ebaade86da8c2155a6cfa662ad0280","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102912928209440768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6846,"pr_state":"open","pr_sha":"99ee68d823b5d9607d6d356b0b4ee40d3ed8f7bd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-24T00:08:06Z","started_at":"2026-09-24T00:08:06Z","completed_at":"2026-09-24T02:26:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T02:36:52.552504385Z","timing_metadata_checked_at":"2026-09-24T02:36:52.552504385Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T02:36:52.555990473Z","region_metadata_checked_at":"2026-09-24T02:36:52.555990473Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"aeb4d72c6550a40c333dfc498f293f583306f735d362939343ce5134018de3c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102912928209440768","occurred_at":"2026-09-24T00:08:06Z","failed_at":"alert","signature_id":"6e5e5e7d518f60d91ecb09219367313b7d859bc18594e43d3b6d1d6226085c18","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-24T02:05:30Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j9440768-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-rbr8w\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-rbr8w (namespace arobit) on cluster ci01-j9440768-svc has averaged 126.2% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]},{"environment":"int","summary":{"total_runs":6,"failed_runs":1,"runs_with_occurrences":1,"runs_with_failure_pattern_matches":1,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":5,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103255388148207616","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-24T22:48:55Z","started_at":"2026-09-24T22:48:55Z","completed_at":"2026-09-25T01:11:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T01:21:08.272118149Z","timing_metadata_checked_at":"2026-09-25T01:21:08.272118149Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T01:21:08.27497444Z","region_metadata_checked_at":"2026-09-25T01:21:08.27497444Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103193820085620736","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-24T18:44:16Z","started_at":"2026-09-24T18:44:16Z","completed_at":"2026-09-24T21:07:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T21:19:12.123814081Z","timing_metadata_checked_at":"2026-09-24T21:19:12.123814081Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T21:19:12.12691427Z","region_metadata_checked_at":"2026-09-24T21:19:12.12691427Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103136634483511296","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T14:57:02Z","started_at":"2026-09-24T14:57:02Z","completed_at":"2026-09-24T17:19:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T17:27:42.046656674Z","timing_metadata_checked_at":"2026-09-24T17:27:42.046656674Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T17:27:42.049676164Z","region_metadata_checked_at":"2026-09-24T17:27:42.049676164Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"062efc54b7e9f728d88362207338d06fca92457ec445a1a47d9cda87e968c1b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103136634483511296","occurred_at":"2026-09-24T14:57:02Z","failed_at":"e2e","signature_id":"c6250a5a334f195a565bf422264939eadf384e54a5927ab71926e7b2cd23c48c","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:155]: cluster viability check failed\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00158c4c8\u003e: \n VerifyBasicAccess failed: failed to list services: Get \"https://api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net:443/api/v1/namespaces/default/services\": dial tcp: lookup api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.129.166.38:32939-\u003e172.30.0.10:53: i/o timeout\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.129.166.38:32939-\u003e172.30.0.10:53: i/o timeout\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:155]: cluster viability check failed\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00158c4c8\u003e: \n VerifyBasicAccess failed: failed to list services: Get \"https://api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net:443/api/v1/namespaces/default/services\": dial tcp: lookup api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.129.166.38:32939-\u003e172.30.0.10:53: i/o timeout\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.129.166.38:32939-\u003e172.30.0.10:53: i/o timeout\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9bf4f3ad9b586932b28d6b547ac6a878bc489620fc6853c6e80a7ffdde8e88d1","failure_pattern":"VerifyBasicAccess failed: failed to list services: Get \"\u003curl\u003e","search_query":"VerifyBasicAccess failed: failed to list services: Get \"https://api"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9bf4f3ad9b586932b28d6b547ac6a878bc489620fc6853c6e80a7ffdde8e88d1"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103088390625824768","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-24T11:45:19Z","started_at":"2026-09-24T11:45:19Z","completed_at":"2026-09-24T13:59:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T14:05:54.867738356Z","timing_metadata_checked_at":"2026-09-24T14:05:54.867738356Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T14:05:54.870777546Z","region_metadata_checked_at":"2026-09-24T14:05:54.870777546Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103037130543468544","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-24T08:21:39Z","started_at":"2026-09-24T08:21:39Z","completed_at":"2026-09-24T10:54:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T10:57:51.123699852Z","timing_metadata_checked_at":"2026-09-24T10:57:51.123699852Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T10:57:51.126469243Z","region_metadata_checked_at":"2026-09-24T10:57:51.126469243Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2102979541436731392","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-24T04:32:47Z","started_at":"2026-09-24T04:32:47Z","completed_at":"2026-09-24T06:52:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T06:56:52.244048315Z","timing_metadata_checked_at":"2026-09-24T06:56:52.244048315Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T06:56:52.246843605Z","region_metadata_checked_at":"2026-09-24T06:56:52.246843605Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]},{"environment":"prod","summary":{"total_runs":5,"failed_runs":5,"runs_with_occurrences":5,"runs_with_failure_pattern_matches":5,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103267802377883648","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T23:38:15Z","started_at":"2026-09-24T23:38:15Z","completed_at":"2026-09-25T02:06:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T02:13:49.826804531Z","timing_metadata_checked_at":"2026-09-25T02:13:49.826804531Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T02:13:49.83017162Z","region_metadata_checked_at":"2026-09-25T02:13:49.83017162Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"1b0ff4b6c8b977178a032ddfcbc1acbcb0e3181779e0a9617667df4f33a9312e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103267802377883648","occurred_at":"2026-09-24T23:38:15Z","failed_at":"e2e","signature_id":"5159cce0088044d8bc192dbe0cedf4bcbebc146393b1f4ddad44f3a9d431c657","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxjdxchm and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-lthncjjflrt5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc001150fb0\u003e: \n storage account log verification failed\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxjdxchm and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-lthncjjflrt5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc001150fb0\u003e: \n storage account log verification failed\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed"}},{"row_id":"58584bc179225602c7e7f656c99ebbf1f0d42a72dce42e2f0c64f125d3b2afcd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103267802377883648","occurred_at":"2026-09-24T23:38:15Z","failed_at":"e2e","signature_id":"bd9b3b7f5ce7275bd4f32fb355609d9038fed94688be48bf618a3c2e0b9ca102","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00088a9a0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-6bxr9l\" in resourcegroup=\"rg-np-noedge-6bxr9l-s8q7zdxcxq2b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2b9dc29e-fe8f-45b4-b8b8-139dfe59a7f6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2b9dc29e-fe8f-45b4-b8b8-139dfe59a7f6\",\n \"name\": \"2b9dc29e-fe8f-45b4-b8b8-139dfe59a7f6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T01:01:50.31869719Z\",\n \"endTime\": \"2026-09-25T01:20:50.832464334Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00088a9a0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-6bxr9l\" in resourcegroup=\"rg-np-noedge-6bxr9l-s8q7zdxcxq2b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2b9dc29e-fe8f-45b4-b8b8-139dfe59a7f6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2b9dc29e-fe8f-45b4-b8b8-139dfe59a7f6\",\n \"name\": \"2b9dc29e-fe8f-45b4-b8b8-139dfe59a7f6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T01:01:50.31869719Z\",\n \"endTime\": \"2026-09-25T01:20:50.832464334Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"aae18a0e590dfdd9c57a2e6c035321350b1312718930d67aa57902962c067ae9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103267802377883648","occurred_at":"2026-09-24T23:38:15Z","failed_at":"e2e","signature_id":"a7e14597b86188a7681e872f2938a0cc446f731377f16c3c6f83c2cc272f9397","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-ml64wq with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013c02a0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-ml64wq: failed waiting for cluster=\"np-version-upgrade-cluster-ml64wq\" in resourcegroup=\"rg-np-version-upgrade-ml64wq-zkbzk7hvlvck\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\",\n \"name\": \"2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T00:48:56.872501334Z\",\n \"endTime\": \"2026-09-25T01:18:00.947451943Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-ml64wq with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013c02a0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-ml64wq: failed waiting for cluster=\"np-version-upgrade-cluster-ml64wq\" in resourcegroup=\"rg-np-version-upgrade-ml64wq-zkbzk7hvlvck\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\",\n \"name\": \"2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T00:48:56.872501334Z\",\n \"endTime\": \"2026-09-25T01:18:00.947451943Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e3e491019ecf3f140e87c882b7d45ae9d3071ed1df24cc639d33c7a9d0e29087","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","e3e491019ecf3f140e87c882b7d45ae9d3071ed1df24cc639d33c7a9d0e29087"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T23:00:14Z","started_at":"2026-09-24T23:00:14Z","completed_at":"2026-09-25T01:13:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T01:21:08.413605009Z","timing_metadata_checked_at":"2026-09-25T01:21:08.413605009Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T01:21:08.416356701Z","region_metadata_checked_at":"2026-09-25T01:21:08.416356701Z"},"failed_at":["e2e"],"failed_test_count":13,"occurrences":[{"row_id":"0c3895616f6dc162c055df7320c686c5e185463bb65e94c414b7edecb64de2ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"369b11784caf993a1266d4a2191af0129727f06deaa23b4c15d665538f183d95","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0010a2930\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-grftwltzcrgn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0010a2930\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-grftwltzcrgn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"1afd47edd9e3e715b81dc5cd5545f76b6508414423f3cce9bd5bfb00dce6e6da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"6d276974ce3c3d3fd4a182577c36d577de093c2c2e4513abb2edf58915543439","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-854hq8s8lwp6/cluster-nightly-5-1-npws6l should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ed0ea0\u003e: \n failed to create HCP cluster cluster-nightly-5-1-npws6l: failed waiting for cluster=\"cluster-nightly-5-1-npws6l\" in resourcegroup=\"rg-nightly-5-1-854hq8s8lwp6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5427e4d8-cab2-43c9-ac23-7d670008e23b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5427e4d8-cab2-43c9-ac23-7d670008e23b\",\n \"name\": \"5427e4d8-cab2-43c9-ac23-7d670008e23b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T23:33:11.266527055Z\",\n \"endTime\": \"2026-09-25T00:02:18.550348976Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-854hq8s8lwp6/cluster-nightly-5-1-npws6l should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ed0ea0\u003e: \n failed to create HCP cluster cluster-nightly-5-1-npws6l: failed waiting for cluster=\"cluster-nightly-5-1-npws6l\" in resourcegroup=\"rg-nightly-5-1-854hq8s8lwp6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5427e4d8-cab2-43c9-ac23-7d670008e23b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5427e4d8-cab2-43c9-ac23-7d670008e23b\",\n \"name\": \"5427e4d8-cab2-43c9-ac23-7d670008e23b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T23:33:11.266527055Z\",\n \"endTime\": \"2026-09-25T00:02:18.550348976Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3cdcd1af82e9b7b34504522c99c5b113af32b0f3e2c9414d5a5fc6c4f758bae3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"a0d8bdae7638bd5124c215d98805edbc939d87889da4c9f0c02457aecd6221df","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxdxtlkr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-zhq2pjd4clbv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc00066d200\u003e: \n storage account log verification failed\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxdxtlkr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-zhq2pjd4clbv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc00066d200\u003e: \n storage account log verification failed\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed"}},{"row_id":"40b3667cdb94b7d5f1cfceed4f77484ef5315c8f5328b3097d1e7a2182c256b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"d8fdb12d282c5253e72136a3836928bc9db7e3e73c7eec5cc70b96bb18cad7d3","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000c772d8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-24-142420, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000c772d8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-24-142420, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"4de00383da021ef5b4826627680ada7e1bce2aa923a9d2dcb7f05c697100a09c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"6e1813acc449a6535ee059b4dc15d561d04689edaf2ac0927414e732fa390dd3","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d9eba0\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ftgp6q\" in resourcegroup=\"rg-np-version-upgrade-ftgp6q-8slz9vtsg7dc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-ftgp6q-8slz9vtsg7dc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-ftgp6q/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d9eba0\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ftgp6q\" in resourcegroup=\"rg-np-version-upgrade-ftgp6q-8slz9vtsg7dc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-ftgp6q-8slz9vtsg7dc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-ftgp6q/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"5e391d3496716438731882154bb2a524129aef8413bff3eab814a49218ca3229","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"163ba41eff7851369f9490bc51639528b5ab45ae39f71f38a3bbb8b825ec7e1d","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0002b1230\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-cp45sss7fz6v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0002b1230\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-cp45sss7fz6v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"708afdd8eab4ba701a6e0c93260dbbaf6cea67a75aebeaf54c5fef7640f5729a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"6d281a26d8ac50b58188d06190db9e64800fa423362a34399d5e8db878ac7e06","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b24be0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-qvrt9xzb9h7m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/05a99209-7724-468c-8d11-789afc14d5d3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/05a99209-7724-468c-8d11-789afc14d5d3\",\n \"name\": \"05a99209-7724-468c-8d11-789afc14d5d3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T23:58:35.632366587Z\",\n \"endTime\": \"2026-09-25T00:17:38.767686221Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b24be0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-qvrt9xzb9h7m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/05a99209-7724-468c-8d11-789afc14d5d3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/05a99209-7724-468c-8d11-789afc14d5d3\",\n \"name\": \"05a99209-7724-468c-8d11-789afc14d5d3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T23:58:35.632366587Z\",\n \"endTime\": \"2026-09-25T00:17:38.767686221Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7142eceeb6195e57f9a1bf07ee45245be5817a802c395762f4c6aa814b435ce9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"2c2448420f0585ced642c7a920d0fe02a87408a0b9d29f4ef0081d2ced376382","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c9720\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-r2rlkr\" resourcegroup=\"rg-np-dg-minor-r2rlkr-ngfwrz48zjxx\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-r2rlkr-ngfwrz48zjxx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-r2rlkr/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c9720\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-r2rlkr\" resourcegroup=\"rg-np-dg-minor-r2rlkr-ngfwrz48zjxx\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-r2rlkr-ngfwrz48zjxx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-r2rlkr/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"7e9b3b1b26c86a21fe4037fb14ec82d9c9c213f3bcf92529544718e0094a110d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"7e29c61c5ec9623a1d7631cc8d8c2bf09662bec493f2b1e0c6761a8e6df9e943","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c260e0\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-6rqpmb\" in resourcegroup=\"rg-np-skip-minor-6rqpmb-k2c9cm8vc6c5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-6rqpmb-k2c9cm8vc6c5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-6rqpmb/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c260e0\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-6rqpmb\" in resourcegroup=\"rg-np-skip-minor-6rqpmb-k2c9cm8vc6c5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-6rqpmb-k2c9cm8vc6c5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-6rqpmb/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a135f52e8ea105138aaa09909dc685e51e13fbf9b4ba3e58022d4894fa29f74c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"62770c4f0e621d8962a43577db753baffbfa217cfa5c51a5444cc20ba8dcbaa1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-d25jl2\" for cluster \"cluster-nightly-4-20-d25jl2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0a140\u003e: \n failed to create NodePool np-d25jl2: failed starting nodepool creation \"np-d25jl2\" for cluster \"cluster-nightly-4-20-d25jl2\" in resourcegroup=\"rg-nightly-4-20-kl7rft6vttsr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-kl7rft6vttsr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-d25jl2/nodePools/np-d25jl2\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-d25jl2\" for cluster \"cluster-nightly-4-20-d25jl2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0a140\u003e: \n failed to create NodePool np-d25jl2: failed starting nodepool creation \"np-d25jl2\" for cluster \"cluster-nightly-4-20-d25jl2\" in resourcegroup=\"rg-nightly-4-20-kl7rft6vttsr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-kl7rft6vttsr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-d25jl2/nodePools/np-d25jl2\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d264ca2fd85e28f65b379c17478ce72e36c12d53b5728cbe6910fc297fae899f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"78618845cda7ac86dc1a7ff167f19e01de1f527c7ec62a7cbacf618e8f8a10e3","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0013b2150\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8g8c7wprtmtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0013b2150\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8g8c7wprtmtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d88a65f42998d46f6c324da34de71246c797b1ad20f813feeee2115f77124c9c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"0856a5b7612ceaf27aef57335208fdb96fec3578c31cded0af090414cbc459ac","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0017055c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-vj7bqc7l746k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0017055c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-vj7bqc7l746k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"eeff63182d185fe43e5083fda604b8ba1da7b633b447b86d6747aaf2ef46d986","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"39cc51f3ed9337761bd1c79d82ca52214f6889badaaab5d696c36fd31637d8b2","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002802840\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qrllhzxjl5qf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002802840\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qrllhzxjl5qf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":13,"failure_pattern_ids":["33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62"],"matched_occurrences":13,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103224303930576896","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T20:45:24Z","started_at":"2026-09-24T20:45:24Z","completed_at":"2026-09-24T23:11:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T23:25:07.242246468Z","timing_metadata_checked_at":"2026-09-24T23:25:07.242246468Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T23:25:07.250814741Z","region_metadata_checked_at":"2026-09-24T23:25:07.250814741Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0b31300fdb25b8d6c1f8d1696189dfe589636f2b049cecc3b198d2f0b01315ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103224303930576896","occurred_at":"2026-09-24T20:45:24Z","failed_at":"e2e","signature_id":"3112eb82261054cccf2006078302df6fc47e6d756350b2f95e0fd2f5e3283de9","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011137a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-vhxnnlnn4h55\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8121b572-5dd9-49c7-9e1e-bb37581297c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8121b572-5dd9-49c7-9e1e-bb37581297c6\",\n \"name\": \"8121b572-5dd9-49c7-9e1e-bb37581297c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T20:58:20.871989951Z\",\n \"endTime\": \"2026-09-24T21:17:23.599073745Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011137a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-vhxnnlnn4h55\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8121b572-5dd9-49c7-9e1e-bb37581297c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8121b572-5dd9-49c7-9e1e-bb37581297c6\",\n \"name\": \"8121b572-5dd9-49c7-9e1e-bb37581297c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T20:58:20.871989951Z\",\n \"endTime\": \"2026-09-24T21:17:23.599073745Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fc961940d64263efffeba073e6c91f29bf52aeac6b8b6aad4a4450c7203a5ecf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103224303930576896","occurred_at":"2026-09-24T20:45:24Z","failed_at":"e2e","signature_id":"5c46d71a0a4a0451e999a028441816085c5b759ae39c7276fb744cd76b6b57a1","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxj5dtdr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-v7bfrx7ddjpw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc000e01080\u003e: \n storage account log verification failed\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxj5dtdr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-v7bfrx7ddjpw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc000e01080\u003e: \n storage account log verification failed\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103171230415720448","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T17:14:30Z","started_at":"2026-09-24T17:14:30Z","completed_at":"2026-09-24T19:59:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T20:06:14.429368554Z","timing_metadata_checked_at":"2026-09-24T20:06:14.429368554Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T20:06:14.43232214Z","region_metadata_checked_at":"2026-09-24T20:06:14.43232214Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"98cd2c547e82228193888e5ee5ab46b117d3cfc6e888c0eff38566aa9594daa4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103171230415720448","occurred_at":"2026-09-24T17:14:30Z","failed_at":"e2e","signature_id":"849d9e776f46a941d1482cb6eef9a594d740471e89a56ba737d225b1ff968668","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxbkcvlx and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-q8hblx6hxzvd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc0009ce940\u003e: \n storage account log verification failed\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxbkcvlx and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-q8hblx6hxzvd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc0009ce940\u003e: \n storage account log verification failed\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed"}},{"row_id":"c302cf72d9658711c38cd1395ef7709026a3bbc9e4ae9c2ec83bbd5bca32d208","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103171230415720448","occurred_at":"2026-09-24T17:14:30Z","failed_at":"e2e","signature_id":"0b0108efd360078eb545d9c8493397d07692133492d39ba4345c2659c856781d","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node a8y5x1l0f9x5u8y-cordon-np-8kx7p-97f6x\nUnexpected error:\n \u003c*errors.StatusError | 0xc00103ab40\u003e: \n Operation cannot be fulfilled on nodes \"a8y5x1l0f9x5u8y-cordon-np-8kx7p-97f6x\": the object has been modified; please apply your changes to the latest version and try again\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node a8y5x1l0f9x5u8y-cordon-np-8kx7p-97f6x\nUnexpected error:\n \u003c*errors.StatusError | 0xc00103ab40\u003e: \n Operation cannot be fulfilled on nodes \"a8y5x1l0f9x5u8y-cordon-np-8kx7p-97f6x\": the object has been modified; please apply your changes to the latest version and try again\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1c2f7c1ada887c9974dadc757d0c5e8d6fb57702022a1a3603efe040da29e12d","failure_pattern":"Operation cannot be fulfilled on nodes \"\u003cnode\u003e\": the object has been modified; please apply your changes to the latest version and try again","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["1c2f7c1ada887c9974dadc757d0c5e8d6fb57702022a1a3603efe040da29e12d","a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T02:00:13Z","started_at":"2026-09-24T02:00:13Z","completed_at":"2026-09-24T04:01:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T04:09:52.895699799Z","timing_metadata_checked_at":"2026-09-24T04:09:52.895699799Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T04:09:52.899371787Z","region_metadata_checked_at":"2026-09-24T04:09:52.899371787Z"},"failed_at":["e2e"],"failed_test_count":8,"occurrences":[{"row_id":"091175b13bb91cb6d967c682b864bc3411362bfe3a2d716a98cf5c8ac9e70a4c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"e328078175fb41c8dfbd4578e31c90a7c72998a5382576bff48a3e7cd16f791f","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010aab40\u003e: \n failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-jh96j8fnch7m\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-jh96j8fnch7m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010aab40\u003e: \n failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-jh96j8fnch7m\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-jh96j8fnch7m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"163fc4b606445f2d4999df0fc14cdc847a108b9259ce6a0cf19ffd1a9960eb24","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"0f42ca5bdd5424ebd462017e995420f979246e19a7f68c1a45f2e182770b37a3","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ab7ea0\u003e: \n failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-r6szljgv5m4p\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-r6szljgv5m4p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ab7ea0\u003e: \n failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-r6szljgv5m4p\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-r6szljgv5m4p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"466be3bdfe781bd637f4f8b20ae2c0e7b28abd1d0d2405cdf43972e33a853ec2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"f1db6949bdfa0c436485b175eee7c33e360c184472692055a39f14766a6685ef","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009c0220\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-jrbwgvcxzd5v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-jrbwgvcxzd5v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009c0220\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-jrbwgvcxzd5v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-jrbwgvcxzd5v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"767d7856e6472a8fb173fd73bc2c0d883c63dfdfa87eea6855fee532d2ed82ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"ae9cfa1cabd766fc0ba51cde7820c4aff0d9b5fe7146c215e79849948da08c5d","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc001321160\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-l2xrb7ls9tkm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e3486a0-a8be-491f-b7ba-de40bdc6a4c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e3486a0-a8be-491f-b7ba-de40bdc6a4c5\",\n \"name\": \"4e3486a0-a8be-491f-b7ba-de40bdc6a4c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T03:02:54.530721397Z\",\n \"endTime\": \"2026-09-24T03:21:54.293711104Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc001321160\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-l2xrb7ls9tkm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e3486a0-a8be-491f-b7ba-de40bdc6a4c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e3486a0-a8be-491f-b7ba-de40bdc6a4c5\",\n \"name\": \"4e3486a0-a8be-491f-b7ba-de40bdc6a4c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T03:02:54.530721397Z\",\n \"endTime\": \"2026-09-24T03:21:54.293711104Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-l2xrb7ls9tkm\" to finish creating: GET https://management"}},{"row_id":"b7f1906845926f7ab212ae6fc68fcaa2b34d6c0f2c70213a5aab35daf5aaa17d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"dfda133e06b8ddc58347c148dd004db522d35f830c18a3e53b5d3cde82d08ea0","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005a06e0\u003e: \n failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-vzmb26shnztz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-vzmb26shnztz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005a06e0\u003e: \n failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-vzmb26shnztz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-vzmb26shnztz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"be4eec1afabd97970839b71b2b484775035a6e571f372184b3fc3f2a9d672405","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"3c9ac5911363cbb207f63f2d230928ca57009144d5f790aadca90edd77cd8e2e","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e90d80\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-pc4xbw2wvdgz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-pc4xbw2wvdgz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e90d80\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-pc4xbw2wvdgz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-pc4xbw2wvdgz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"c55920c7c29064d9a86c53f29121e513a344361b9419f16b16d10b2eed97babb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"a043abfb4a137ba5c7fb99a1284be7cfe198204b214476f6fd57e00fee845d3a","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0015fdec0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-gk9hkpv4f4bq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0015fdec0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-gk9hkpv4f4bq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"fd7732f1b2077822f018d14a789f936569ad1abdf47c8ce1c5404b10c2d276e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"fa688a48213278aa49f8f4617b7ea8d841d18133d8772e5ee0c28401f6d5f777","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c12d20\u003e: \n failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fttxlklms5k2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-fttxlklms5k2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c12d20\u003e: \n failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fttxlklms5k2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-fttxlklms5k2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]},{"environment":"stg","summary":{"total_runs":8,"failed_runs":8,"runs_with_occurrences":8,"runs_with_failure_pattern_matches":8,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T23:00:14Z","started_at":"2026-09-24T23:00:14Z","completed_at":"2026-09-25T00:56:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T01:10:56.779031513Z","timing_metadata_checked_at":"2026-09-25T01:10:56.779031513Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T01:10:56.782813401Z","region_metadata_checked_at":"2026-09-25T01:10:56.782813401Z"},"failed_at":["e2e"],"failed_test_count":13,"occurrences":[{"row_id":"1252f0b20ca354ed5258d34a94a120442d3ef5fd4405988bf77b0c3ef5315332","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"ccb025795b93e958b7aae9d388bb9cdb57b9a396064e500aff59a625a9f0e777","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-wllchtq6tspw/cluster-nightly-5-1-xqrkdk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000614980\u003e: \n failed to create HCP cluster cluster-nightly-5-1-xqrkdk: failed waiting for cluster=\"cluster-nightly-5-1-xqrkdk\" in resourcegroup=\"rg-nightly-5-1-wllchtq6tspw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8d90a29e-3743-4f19-9012-64f6e3adf3af\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8d90a29e-3743-4f19-9012-64f6e3adf3af\",\n \"name\": \"8d90a29e-3743-4f19-9012-64f6e3adf3af\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T23:29:29.214562248Z\",\n \"endTime\": \"2026-09-24T23:58:29.739929622Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-wllchtq6tspw/cluster-nightly-5-1-xqrkdk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000614980\u003e: \n failed to create HCP cluster cluster-nightly-5-1-xqrkdk: failed waiting for cluster=\"cluster-nightly-5-1-xqrkdk\" in resourcegroup=\"rg-nightly-5-1-wllchtq6tspw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8d90a29e-3743-4f19-9012-64f6e3adf3af\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8d90a29e-3743-4f19-9012-64f6e3adf3af\",\n \"name\": \"8d90a29e-3743-4f19-9012-64f6e3adf3af\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T23:29:29.214562248Z\",\n \"endTime\": \"2026-09-24T23:58:29.739929622Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d523a0d2817fdbba88731dd1892c07aa03075f35ab21021c384553c8aa80da9e","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1c90978edf4e060819afa1d516cb5161d011fdd9dbb92081aabece3d983287ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"2c832c99908ab7e4af98af64b3ea848748c2d5da97c25a86fe752084f15d42f0","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-cf4l7z with version 4.21.0-0.nightly-multi-2026-09-24-033657\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00142c760\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-cf4l7z: failed waiting for cluster=\"np-version-upgrade-cluster-cf4l7z\" in resourcegroup=\"rg-np-version-upgrade-cf4l7z-c8gcs9n4rck7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/58578399-7a04-4d7e-9fff-65b3ebeb1ba4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/58578399-7a04-4d7e-9fff-65b3ebeb1ba4\",\n \"name\": \"58578399-7a04-4d7e-9fff-65b3ebeb1ba4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T23:42:40.568397466Z\",\n \"endTime\": \"2026-09-25T00:11:49.734192333Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-cf4l7z with version 4.21.0-0.nightly-multi-2026-09-24-033657\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00142c760\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-cf4l7z: failed waiting for cluster=\"np-version-upgrade-cluster-cf4l7z\" in resourcegroup=\"rg-np-version-upgrade-cf4l7z-c8gcs9n4rck7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/58578399-7a04-4d7e-9fff-65b3ebeb1ba4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/58578399-7a04-4d7e-9fff-65b3ebeb1ba4\",\n \"name\": \"58578399-7a04-4d7e-9fff-65b3ebeb1ba4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T23:42:40.568397466Z\",\n \"endTime\": \"2026-09-25T00:11:49.734192333Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"41dadc374b6fe2625e6ee7acdf8fbc9a991cbd05173905d55b61bd73224d1bb8","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2e95b9691d0647fdb69a8e1036c28709731f1334b9d346a8b5f2e89e8364ec5f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"27740ed35f7486949fc1f199355bb1882c9f8cb9081d779fb41653a80527acd4","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0019711d0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7dp4spk72w92/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0019711d0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7dp4spk72w92/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"67538d3ab09af8a92a9e0696fce937fc57c3c02155d7ad81e7dd90017e824994","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"bb492d03ce36d043a7737e16613de6f3e8eaf5e4ffe34f36ac441cff010efa9a","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0016823c0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-24-142420, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0016823c0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-24-142420, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"6869201b85d0e8e6c3606a2bb97cd7af3cfda132dace0ada277079146c3290b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"691dfc6d9fb2291ffc32616264d74d1f43851ef85e826708db94455617cad251","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001566b10\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-z76xpv9lv7mg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001566b10\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-z76xpv9lv7mg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"6f059471b80253f98fb4c068c3add697440b44c145162ffa0e574dcc63a155e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"1b2eae294949b5b68cbe342df9ea515c496f1ce1130bf8c19dff2578413bca2c","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f5c1c0\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-k5htqv\" in resourcegroup=\"rg-np-skip-minor-k5htqv-87tsfgngkbxz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-k5htqv-87tsfgngkbxz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-k5htqv/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f5c1c0\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-k5htqv\" in resourcegroup=\"rg-np-skip-minor-k5htqv-87tsfgngkbxz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-k5htqv-87tsfgngkbxz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-k5htqv/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"7ad45f9dac5c5307dbc0eae44525e641821da8aefe7fa721e54f25c9a0218999","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"c5e7a15d5897753355cc4005fa9bf61ec537390259186723bdfc09da6d61808d","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0021e3bc0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-cccvsc896tfj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0021e3bc0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-cccvsc896tfj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a6b8adaca07db16b11437203ef10520214a4e06fd47b381bfd08ba1530394102","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"9c871760b90c87c480ead77cf16197e08922039d3cb3658d834ce5592bb2917a","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0011064b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-6qptj87wv4cv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0011064b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-6qptj87wv4cv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b29431ca1f69d5e00b8ef4c952aff681a315899ecca5ef946878005b8e34660f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"2499142f7ad8e903e0f48a80ca5cf2771b399a595df8551871acaf3736c67691","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-g6wkqd\" for cluster \"cluster-nightly-4-20-g6wkqd\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000926600\u003e: \n failed to create NodePool np-g6wkqd: failed starting nodepool creation \"np-g6wkqd\" for cluster \"cluster-nightly-4-20-g6wkqd\" in resourcegroup=\"rg-nightly-4-20-2j2bp6ng4gdp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-2j2bp6ng4gdp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-g6wkqd/nodePools/np-g6wkqd\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-g6wkqd\" for cluster \"cluster-nightly-4-20-g6wkqd\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000926600\u003e: \n failed to create NodePool np-g6wkqd: failed starting nodepool creation \"np-g6wkqd\" for cluster \"cluster-nightly-4-20-g6wkqd\" in resourcegroup=\"rg-nightly-4-20-2j2bp6ng4gdp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-2j2bp6ng4gdp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-g6wkqd/nodePools/np-g6wkqd\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b4c49b6c8180b3aa39776019bf0df1431b6027d6cb4ca9f8fc5f9147242a26f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"3b28ac3f4fe19a90ee317ea1737bd43f097cfc518ad408fd1fd876a861e94ad9","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00151a140\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-4pqnxp\" resourcegroup=\"rg-np-dg-minor-4pqnxp-4lnmhzl52tvp\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-4pqnxp-4lnmhzl52tvp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-4pqnxp/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00151a140\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-4pqnxp\" resourcegroup=\"rg-np-dg-minor-4pqnxp-4lnmhzl52tvp\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-4pqnxp-4lnmhzl52tvp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-4pqnxp/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"bcd8393f021e5e8f00a9e99b34020da3502b0d7263469d379b9bdcc4e790bff4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"2138dc55899dce5641331077f3e289aee19252fb625b5f07c0d419ec286f1599","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoebox6npjq8 and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-g4qtcfdblkfs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc000d18d20\u003e: \n storage account log verification failed\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoebox6npjq8 and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-g4qtcfdblkfs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc000d18d20\u003e: \n storage account log verification failed\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"88c34ce3eb4dd39781e8ddf92764a71a4749619137d6e0e581fc858207a4e653","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed"}},{"row_id":"cf99d876bc6fe526aaae348224deb6ffef10e43841a64186ccc0a366660ec8b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"7fbe71528d20c91eedbc5d9a15147ed84d81599aec91f81260d0bbd325e1866b","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0013a3320\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-cgk7n47ncfl4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0013a3320\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-cgk7n47ncfl4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"f8fe60212200f3332c79a1e6c809e2ab37a5585ce58703d8c4786047195850dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"b1a2325b06cdb9a45a78b9f23c355fd6276e541cbba1065d34722c4530f70a01","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d07ae0\u003e: \n failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-xxwmzws6kxxc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85216ecc-6f15-4ee4-9411-ab10183b2cec\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85216ecc-6f15-4ee4-9411-ab10183b2cec\",\n \"name\": \"85216ecc-6f15-4ee4-9411-ab10183b2cec\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T23:13:56.752561651Z\",\n \"endTime\": \"2026-09-24T23:32:59.664565124Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d07ae0\u003e: \n failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-xxwmzws6kxxc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85216ecc-6f15-4ee4-9411-ab10183b2cec\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85216ecc-6f15-4ee4-9411-ab10183b2cec\",\n \"name\": \"85216ecc-6f15-4ee4-9411-ab10183b2cec\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T23:13:56.752561651Z\",\n \"endTime\": \"2026-09-24T23:32:59.664565124Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d5a27e23f87f5c18e786f32d10682561abfa5723896eb83d080ad6377beec74f","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":13,"failure_pattern_ids":["28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","41dadc374b6fe2625e6ee7acdf8fbc9a991cbd05173905d55b61bd73224d1bb8","646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","88c34ce3eb4dd39781e8ddf92764a71a4749619137d6e0e581fc858207a4e653","b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","d523a0d2817fdbba88731dd1892c07aa03075f35ab21021c384553c8aa80da9e","d5a27e23f87f5c18e786f32d10682561abfa5723896eb83d080ad6377beec74f"],"matched_occurrences":13,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T20:07:48Z","started_at":"2026-09-24T20:07:48Z","completed_at":"2026-09-24T20:39:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T20:47:32.187618234Z","timing_metadata_checked_at":"2026-09-24T20:47:32.187618234Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T20:47:32.190201025Z","region_metadata_checked_at":"2026-09-24T20:47:32.190201025Z"},"failed_at":["e2e"],"failed_test_count":59,"occurrences":[{"row_id":"003e99c97103d4c635fd5d59c7ec5c576a4986429af48e3d24c9db34565e1485","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"21ec5f782bbbcebf9c1bb08d8f3860277c036fb88121f67bd7df545279ec079f","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-przs8v with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000736620\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-przs8v: failed starting cluster creation \"np-version-upgrade-cluster-przs8v\" in resourcegroup=\"rg-np-version-upgrade-przs8v-vc4ptfztdpvg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-przs8v-vc4ptfztdpvg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-przs8v\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-przs8v with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000736620\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-przs8v: failed starting cluster creation \"np-version-upgrade-cluster-przs8v\" in resourcegroup=\"rg-np-version-upgrade-przs8v-vc4ptfztdpvg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-przs8v-vc4ptfztdpvg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-przs8v\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"022e26f76b24a00f836b8a804ad69f4991adec0c4e1db1b09085906a56b81f28","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"c1f6def3d468e1d53dc22a82731f7fdf79b4f1e68b13db8097950ff0725b0301","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015650a0\u003e: \n failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-8qxdjfjb7mb2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-8qxdjfjb7mb2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015650a0\u003e: \n failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-8qxdjfjb7mb2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-8qxdjfjb7mb2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"03a0294f91f4d9c0977f8d81a826d2a42f75754830fe2f206d0b9f72499a89d3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"736869058c088afa7ee0e36a8c8df936c6ebd0d3051c6c1de9701a4b9d0d1faa","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-cskmgv\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a70420\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-cskmgv: failed starting cluster creation \"cp-ystream-upgrade-4-21-cskmgv\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-4sk4lx7gbhw6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-4sk4lx7gbhw6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-cskmgv\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-cskmgv\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a70420\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-cskmgv: failed starting cluster creation \"cp-ystream-upgrade-4-21-cskmgv\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-4sk4lx7gbhw6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-4sk4lx7gbhw6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-cskmgv\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"0b52c55879b1c4a96d762a3a014dec5bb929352625f3b6b6634fecfebc433fc1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"3111af3121a7744fec69a5d201dca82460664234a0796e6e4967a0c4f1b2fb94","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015a88e0\u003e: \n failed to create HCP cluster np-dg-minor-zvqbvr: failed starting cluster creation \"np-dg-minor-zvqbvr\" in resourcegroup=\"rg-np-dg-minor-zvqbvr-b979m4957htq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-zvqbvr-b979m4957htq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-zvqbvr\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015a88e0\u003e: \n failed to create HCP cluster np-dg-minor-zvqbvr: failed starting cluster creation \"np-dg-minor-zvqbvr\" in resourcegroup=\"rg-np-dg-minor-zvqbvr-b979m4957htq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-zvqbvr-b979m4957htq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-zvqbvr\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"0cb63416e7110a502225e93c000e88d418b89bfca05d3a9434165a08b2d981bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"847f7b17102dc7090d588a9d72db0f68b89cbbd60c61d629ca5d29b7d31645cd","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0017589f0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: NoRegisteredProviderFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"NoRegisteredProviderFound\",\n \"message\": \"No registered resource provider found for location 'westus3' and API version '2024-06-10-preview' for type 'locations/hcpOpenShiftVersions'. The supported api-versions are '2026-09-01-preview, 2026-06-30-preview, 2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2, uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0017589f0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: NoRegisteredProviderFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"NoRegisteredProviderFound\",\n \"message\": \"No registered resource provider found for location 'westus3' and API version '2024-06-10-preview' for type 'locations/hcpOpenShiftVersions'. The supported api-versions are '2026-09-01-preview, 2026-06-30-preview, 2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2, uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"68cf65637733da6389b66f0d98a7fd646026fde9e15cc697ce29959bae80ec43","failure_pattern":"ERROR CODE: NoRegisteredProviderFound; detail message No registered resource provider found for location 'westus3' and API version '2024-06-10-preview' for type 'locations/hcpOpenShiftVersions'. The supported api-versions are '2026-09-01-preview, 2026-06-30-preview, 2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2, uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: NoRegisteredProviderFound"}},{"row_id":"149abb28c2aefe3d91c7e4add5051b3773a38aeb60092f61fd252ae6e4fd2976","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"15f5f6ae828a46047c858aa69ce37554fa6dd71efe1a2cb7544916905c1ac2ff","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-4t8l4tgxv9hz/cluster-candidate-4-21-2n8mmh should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00092c560\u003e: \n failed to create HCP cluster cluster-candidate-4-21-2n8mmh: failed starting cluster creation \"cluster-candidate-4-21-2n8mmh\" in resourcegroup=\"rg-candidate-4-21-4t8l4tgxv9hz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-4t8l4tgxv9hz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-2n8mmh\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-4t8l4tgxv9hz/cluster-candidate-4-21-2n8mmh should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00092c560\u003e: \n failed to create HCP cluster cluster-candidate-4-21-2n8mmh: failed starting cluster creation \"cluster-candidate-4-21-2n8mmh\" in resourcegroup=\"rg-candidate-4-21-4t8l4tgxv9hz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-4t8l4tgxv9hz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-2n8mmh\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"16e5dd40153f3411c325414e9964e177c518aaefe601ef621c1270bfcb532b9a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"09ed45becaf29978462b1c3b2ec63dbc0323dfba597e299f51b743ca27d93962","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011587a0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-dkc6688q7l7z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-dkc6688q7l7z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011587a0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-dkc6688q7l7z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-dkc6688q7l7z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"18141ef0dc7b33dc20b31f414212623e9ce2c745599e336a3a37730a2d602f46","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"27faee0e8e7daace9345589564265c839002d698d5b6bbaadbbd0fa8335b0ea3","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c722a0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-fbqvn4k6lw4t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-fbqvn4k6lw4t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c722a0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-fbqvn4k6lw4t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-fbqvn4k6lw4t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"1c168bcbfaef9ef4cef499342ab724f70b2f15bfb3a2bafc8a09836d851fa333","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"d8bdf92b2e9204714596bc6a2a6ce29e52d30033a7f33ed1bebc44c552ce7396","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-hpp6c5sdhdrk/cluster-candidate-5-0-9n8h68 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc0320\u003e: \n failed to create HCP cluster cluster-candidate-5-0-9n8h68: failed starting cluster creation \"cluster-candidate-5-0-9n8h68\" in resourcegroup=\"rg-candidate-5-0-hpp6c5sdhdrk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-hpp6c5sdhdrk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-9n8h68\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-hpp6c5sdhdrk/cluster-candidate-5-0-9n8h68 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc0320\u003e: \n failed to create HCP cluster cluster-candidate-5-0-9n8h68: failed starting cluster creation \"cluster-candidate-5-0-9n8h68\" in resourcegroup=\"rg-candidate-5-0-hpp6c5sdhdrk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-hpp6c5sdhdrk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-9n8h68\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"22311cad209c9a3987907a810912da78e6e42e6e777c7c06e91d76f7336c0a58","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"eb955423d40836c1dcb045a85bcda0cb03498e55747b2de9e2048325db028477","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f87400\u003e: \n failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-r58424j4l8b9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-r58424j4l8b9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f87400\u003e: \n failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-r58424j4l8b9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-r58424j4l8b9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"2b2a3bdcb47cb88bc21f865311fb725de773524ba3f4209979148ca1bfd88b19","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"c1fb319f0d065f046fccee3d75086273838a8ab728564590b0d0354f67fbb476","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-gbqkd2\" with version 4.22.14 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00149a920\u003e: \n failed to create HCP cluster cluster-zstream-4-22-gbqkd2: failed starting cluster creation \"cluster-zstream-4-22-gbqkd2\" in resourcegroup=\"rg-zstream-upgrade-4-22-nqb6nvgsb7ch\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-nqb6nvgsb7ch/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-gbqkd2\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-gbqkd2\" with version 4.22.14 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00149a920\u003e: \n failed to create HCP cluster cluster-zstream-4-22-gbqkd2: failed starting cluster creation \"cluster-zstream-4-22-gbqkd2\" in resourcegroup=\"rg-zstream-upgrade-4-22-nqb6nvgsb7ch\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-nqb6nvgsb7ch/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-gbqkd2\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"2e7d3ee5d29ccf9c767165991364b8b56c9aca520cfb5c065e2a3bea4eba9586","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"e2c9151c9266bd39737c0145a83b25202e796e3680f387fe62de4718df836011","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000886a80\u003e: \n failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-5wbvtl2mnmqn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-5wbvtl2mnmqn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000886a80\u003e: \n failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-5wbvtl2mnmqn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-5wbvtl2mnmqn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"3dbf0c0c8161b89bf4bc3bfb5c5d5776443e49c81447f3d89148a84d618da016","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"5603a2702aef8190ce731f49e3f6da6f4c67aef7c8a2f54bf3011d9751ff4929","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-qxkzz7\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a75ba0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-qxkzz7: failed starting cluster creation \"cp-ystream-upgrade-4-23-qxkzz7\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-dzbljwj42bk9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-dzbljwj42bk9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-qxkzz7\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-qxkzz7\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a75ba0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-qxkzz7: failed starting cluster creation \"cp-ystream-upgrade-4-23-qxkzz7\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-dzbljwj42bk9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-dzbljwj42bk9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-qxkzz7\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"4169fc5aeb24179b19d9597239bc4d65b2fd22d6565274feb75263416b865861","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"7eec216fe53eb861071172ca28122a7fdf1495ace014b5cf90792e298e12b594","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000706ce0\u003e: \n failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-qj5qhprdvftt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-qj5qhprdvftt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000706ce0\u003e: \n failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-qj5qhprdvftt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-qj5qhprdvftt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"4bc5e78af77403ee1c21d8df4e88badcb245cc6159c52bb75a47203f61e84858","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"e10cca4cfd9139f298f73a8d53b94d84b35a5fdc8d47d507968b47ecc585cc9c","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001011720\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed starting cluster creation \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-8x4ndpfmx4ct\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8x4ndpfmx4ct/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-delete-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001011720\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed starting cluster creation \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-8x4ndpfmx4ct\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8x4ndpfmx4ct/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-delete-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"503411569dc1f4511e0042a79d6677957209415a130dd1b561c7c8abca2c27a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"aaef8153c4cd154e06971faa50b02660bd9e7ea2bbcc29a2f34394cf6867b48c","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe31e0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-jvstdp6449kc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-jvstdp6449kc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe31e0\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-jvstdp6449kc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-jvstdp6449kc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"507d36b7b5660e4c9db4a80e12cabc5ecaf2b7801859ff8dbaaf344f5cd7ba25","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"53ef031d8e131f3373054e208b7c3fe7af1f963c377b7a456e7885180b60a0d6","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-twdz9c with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001eedc0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-twdz9c: failed starting cluster creation \"np-version-upgrade-cluster-twdz9c\" in resourcegroup=\"rg-np-version-upgrade-twdz9c-bdc7bcn8g67z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-twdz9c-bdc7bcn8g67z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-twdz9c\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-twdz9c with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001eedc0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-twdz9c: failed starting cluster creation \"np-version-upgrade-cluster-twdz9c\" in resourcegroup=\"rg-np-version-upgrade-twdz9c-bdc7bcn8g67z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-twdz9c-bdc7bcn8g67z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-twdz9c\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"57512fce667e1ae066137ff24b808cb0d42d907c1470e8ba33c6a16b6bcc3d92","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"01df788dfb171c0668bb7074e3f88aca94749f84bfc4fdbd1a7fd91e08a7cfa0","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b7e680\u003e: \n failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-d5k7vz9fgmp5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-d5k7vz9fgmp5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b7e680\u003e: \n failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-d5k7vz9fgmp5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-d5k7vz9fgmp5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"5c01d481879b47346818d2cb4067bb37153e753ca9b11e71894aedbe9baab69b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"0d104c19c86ac40a63eda038290eacd0119503471444454de0cfa1dd5d007ef7","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010837c0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-d8bbt9k992fx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-d8bbt9k992fx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010837c0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-d8bbt9k992fx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-d8bbt9k992fx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"6112eb0496ba43cea9dc2db1093fa0edbe64abcee55151923c50a18b40c53daa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"91e6b9daa46d3913f91a3381fa2d7d1c8603487001fbb4805d640eea2c9816bd","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-bk8phf\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fdf180\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-bk8phf: failed starting cluster creation \"cp-ystream-upgrade-4-22-bk8phf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-wzs2hgm7sj86\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-wzs2hgm7sj86/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-bk8phf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-bk8phf\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fdf180\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-bk8phf: failed starting cluster creation \"cp-ystream-upgrade-4-22-bk8phf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-wzs2hgm7sj86\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-wzs2hgm7sj86/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-bk8phf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"66021f337574d2ea932a73322132743ace1c805aded068789d6a0646cc082fa9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"90bf1a4bb0f55805455582cc5075cd286abac466f6ba8106c516801ad7b9f73f","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c456a0\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-8nq9pchcqchg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-8nq9pchcqchg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c456a0\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-8nq9pchcqchg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-8nq9pchcqchg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"6728f2b42b8a5962f3523c458935412820662a71c3cfa4ef1d6395b0ba710047","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"aac0a4fadaa81917b45017334cfcee4d41249c967b1a67dc637c0244f20f2dce","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fdf860\u003e: \n failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-4xdxfr96vsdr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-4xdxfr96vsdr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fdf860\u003e: \n failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-4xdxfr96vsdr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-4xdxfr96vsdr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"6e9e99ba0609174deba3c49534a41726b4c4a93ed684a0a9b5a0f92bb4f90d01","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"ad047b313cd46ff403890ab91105d7e24c9d73411af2aa2c234da58252bd96f6","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f47340\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-s75gnvmnmmhn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-s75gnvmnmmhn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f47340\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-s75gnvmnmmhn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-s75gnvmnmmhn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"6f1eef5acb5f113d63a266233ac5798a8c709522395fa8b0601107f23519792e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"27e7bc710c289477425f6c73f134a06ae089c56c9d0c553da97d5810dc3c16dd","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001121380\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-c25p8tntx7z6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-c25p8tntx7z6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001121380\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-c25p8tntx7z6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-c25p8tntx7z6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"6f68d81014823fb3b32c6e2f7c629789d3dc81d45ee4b19ee477d7c4cb58bf85","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"5cedb6ab199d625ac70604831620fff130147fe82ac076113745bb3622573301","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-mgpwxh with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001978d80\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-mgpwxh: failed starting cluster creation \"np-version-upgrade-cluster-mgpwxh\" in resourcegroup=\"rg-np-version-upgrade-mgpwxh-62rzgbpsmm8q\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mgpwxh-62rzgbpsmm8q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-mgpwxh\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-mgpwxh with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001978d80\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-mgpwxh: failed starting cluster creation \"np-version-upgrade-cluster-mgpwxh\" in resourcegroup=\"rg-np-version-upgrade-mgpwxh-62rzgbpsmm8q\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-mgpwxh-62rzgbpsmm8q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-mgpwxh\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"72bd9ef507a4ea8bf73ecb5f96e8a1a478df830e62a52fde138b0334604eecee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"d52eea2a457782a8ea36566d49d84e7651cc963ab39843912377b9d8c44a04c8","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000825b40\u003e: \n failed to create HCP cluster ea-listtblh6f: failed starting cluster creation \"ea-listtblh6f\" in resourcegroup=\"ea-list-vc7smgt572nt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-vc7smgt572nt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listtblh6f\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000825b40\u003e: \n failed to create HCP cluster ea-listtblh6f: failed starting cluster creation \"ea-listtblh6f\" in resourcegroup=\"ea-list-vc7smgt572nt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-vc7smgt572nt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listtblh6f\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"7aa8de5ecac6570e99a9b2ce9538946a2e033ca160b1634669f2db7025529cd8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"04637e18bfcb3f9e724928ed6c4c1cf581f36315f1bfeccdf3f80802e6356d9d","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-5zxmn6\" with version 4.20.38 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e7afc0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-5zxmn6: failed starting cluster creation \"cluster-zstream-4-20-5zxmn6\" in resourcegroup=\"rg-zstream-upgrade-4-20-87qdz9hglgx4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-87qdz9hglgx4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-5zxmn6\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-5zxmn6\" with version 4.20.38 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e7afc0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-5zxmn6: failed starting cluster creation \"cluster-zstream-4-20-5zxmn6\" in resourcegroup=\"rg-zstream-upgrade-4-20-87qdz9hglgx4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-87qdz9hglgx4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-5zxmn6\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"811eba587761e6509e38c74db0f3eda0f7114fd1404f5fc209d9aaade2cbc22a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"97c7d0cc26d3fed4b372c2504b2ba57618874ad5b3a6f3eeea32900b0bdf71cb","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe4b00\u003e: \n failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-jjx6z2lgm28f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-jjx6z2lgm28f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe4b00\u003e: \n failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-jjx6z2lgm28f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-jjx6z2lgm28f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"88f37308f521fb40c0e78c96b8fd2084f770e6b401cac91ce5a86097f02cac38","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"b24288af3ae287714be949e08bf57141464dd56c19ed7d6236a3c80a3dac299a","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e037e0\u003e: \n failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-dnt66gjnv8ns\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-dnt66gjnv8ns/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e037e0\u003e: \n failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-dnt66gjnv8ns\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-dnt66gjnv8ns/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"8966d1637286b73d79eecdaad99939ec4069fdb19da45597768162350fd87fde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"cc0de1274b9ceffba6b657f32cb1765d623a4c8525822e22472a4854ffdbdfd7","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001cd6e0\u003e: \n failed to create HCP cluster np-downgrade-75gbkn: failed starting cluster creation \"np-downgrade-75gbkn\" in resourcegroup=\"rg-np-downgrade-75gbkn-lrrpbz298xcd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-75gbkn-lrrpbz298xcd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-75gbkn\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001cd6e0\u003e: \n failed to create HCP cluster np-downgrade-75gbkn: failed starting cluster creation \"np-downgrade-75gbkn\" in resourcegroup=\"rg-np-downgrade-75gbkn-lrrpbz298xcd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-75gbkn-lrrpbz298xcd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-75gbkn\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"8c38e9c94c9edf83aac2c4633fbdf9cd087db34641cc32720047b83ec084962b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"9bb7d0520b881857aa31248f17345ac080812b7c304ff8cc09d37fcd8e3025d0","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a68fc0\u003e: \n failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-w7ldgfwkn8pz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-w7ldgfwkn8pz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a68fc0\u003e: \n failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-w7ldgfwkn8pz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-w7ldgfwkn8pz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"8f830562e33bdcb3810f6742f43e84486e207b85073a2435046eaea497960731","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"39d2e978a6cb835985b2a92f5546d92872a269f4e9257aaf79741af5c03d26c5","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-gwg2rx\" with version 4.21.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e86f00\u003e: \n failed to create HCP cluster cluster-zstream-4-21-gwg2rx: failed starting cluster creation \"cluster-zstream-4-21-gwg2rx\" in resourcegroup=\"rg-zstream-upgrade-4-21-sdrclvvxd89m\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-sdrclvvxd89m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-gwg2rx\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-gwg2rx\" with version 4.21.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e86f00\u003e: \n failed to create HCP cluster cluster-zstream-4-21-gwg2rx: failed starting cluster creation \"cluster-zstream-4-21-gwg2rx\" in resourcegroup=\"rg-zstream-upgrade-4-21-sdrclvvxd89m\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-sdrclvvxd89m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-gwg2rx\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"8fbefe9cafb8d5cda22dc2cb7a60643f85150e05b3e32b313bfd0eb5a4f507ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"623139f8f571d2f5ae27e6d75e9b76c6e49ad61954e8cbea14bd347a34071a46","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:91]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000881a00\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-8m56dcvx9lzw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-8m56dcvx9lzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:91]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000881a00\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-8m56dcvx9lzw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-8m56dcvx9lzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"98c1b442f34e1d265514acf44b9517624381a90c5d7d96294c46ddb8ec87a161","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"af5f6dc1f9242795b07b341d7cdab3660235226d8b0ef15223cbd5e68cdb26c1","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e74100\u003e: \n failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-fw89pw968mp7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-fw89pw968mp7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e74100\u003e: \n failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-fw89pw968mp7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-fw89pw968mp7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"9f2be86d6a124c023f38a1fa18757f777eb560024c0380a37ae005ef9febd3c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"e6f223f3f936617d0a8b98f8caf439c033dc69e26b23715f31b40c276bc5de9f","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a798c0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dnzscjfgzjmg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-dnzscjfgzjmg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a798c0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dnzscjfgzjmg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-dnzscjfgzjmg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"a0a707ef49375e26ed24c5cf674201292e6cf7a0ac1d6b76437e3e82c9b49475","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"12e68ce9f4a1569a39aba64a40ff879ce989a6ff69156a6105233cf4609da0d5","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b70d40\u003e: \n failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-nkm54tdkjl42\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-nkm54tdkjl42/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b70d40\u003e: \n failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-nkm54tdkjl42\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-nkm54tdkjl42/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"a2cd0095b20863eb38e1ffa2980a6f437ef8f3d5c62b361467f1554b87b9f0e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"a83f70703c99a3e6b99485097810f7a5ce10697c954f41db95a0c6d9c746e8f3","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-2hgvnr5nrxqw/cluster-candidate-4-22-kqvcgg should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bece20\u003e: \n failed to create HCP cluster cluster-candidate-4-22-kqvcgg: failed starting cluster creation \"cluster-candidate-4-22-kqvcgg\" in resourcegroup=\"rg-candidate-4-22-2hgvnr5nrxqw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-2hgvnr5nrxqw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-22-kqvcgg\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-2hgvnr5nrxqw/cluster-candidate-4-22-kqvcgg should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bece20\u003e: \n failed to create HCP cluster cluster-candidate-4-22-kqvcgg: failed starting cluster creation \"cluster-candidate-4-22-kqvcgg\" in resourcegroup=\"rg-candidate-4-22-2hgvnr5nrxqw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-2hgvnr5nrxqw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-22-kqvcgg\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"a5d7309584573416009d3e8dc47d7a5db843c4e2a1817286bc40883d96132f16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"15fba99f85a72c2b953f1221bc696b9bf82753423dac0791fefd94f24e82a3db","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bc4c20\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-q4fkwqfnvpzh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-q4fkwqfnvpzh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bc4c20\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-q4fkwqfnvpzh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-q4fkwqfnvpzh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"a81cbdff4be1c794abcfde157b480bfd1595cc1159f0ac299c8c4cc10cb65287","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"dfc4c8893a466ebc947358160114e3f5e45b2ef1a49b11beb24e101d649532cd","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-jv7lbnk8wtqv/cluster-candidate-4-20-z76tdl should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017160e0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-z76tdl: failed starting cluster creation \"cluster-candidate-4-20-z76tdl\" in resourcegroup=\"rg-candidate-4-20-jv7lbnk8wtqv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-jv7lbnk8wtqv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-20-z76tdl\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-jv7lbnk8wtqv/cluster-candidate-4-20-z76tdl should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017160e0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-z76tdl: failed starting cluster creation \"cluster-candidate-4-20-z76tdl\" in resourcegroup=\"rg-candidate-4-20-jv7lbnk8wtqv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-jv7lbnk8wtqv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-20-z76tdl\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"a8c796215f25ec4ba14ed0d0abb1f54ef71346bf5e17a4077eb4e28026ff0236","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"e4596221e8bb2095fe0e124bbbec22ca6138f5070eaf0a73880264e0ed679646","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0002196c0\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-4vpw2jpcb2x9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-4vpw2jpcb2x9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0002196c0\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-4vpw2jpcb2x9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-4vpw2jpcb2x9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"ab28f3cea58457cbbd8a52706830081c4ea542de66ac0b5d7046322cb7fef2bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"d030cec66b2abe74848996b1135963ec2f1a5deaa837b7c5222ce37dd6decfbe","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000da3060\u003e: \n failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-58qzvqwc5m7v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-58qzvqwc5m7v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000da3060\u003e: \n failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-58qzvqwc5m7v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-58qzvqwc5m7v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"ac714742da42c4ba72011153f29edbceac42e4cd86e8473ba34df5e235cc12b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"3557fb0d5e8fd6771781276cd1bd6b8e55cef0ac95abd5a7179bcf77ad589c58","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011e97c0\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed starting cluster creation \"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-794wg6jp9b8f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-794wg6jp9b8f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011e97c0\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed starting cluster creation \"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-794wg6jp9b8f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-794wg6jp9b8f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"afbb7e8ce98e33645f2533dbb365542b615db6c34c5686ed710fe0e67540160a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"a322a676fef23495954c29cefc197a2d50b5612cdc27cedbae8d3022650db26a","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c87ba0\u003e: \n failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-7cj42nbzch5r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-7cj42nbzch5r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c87ba0\u003e: \n failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-7cj42nbzch5r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-7cj42nbzch5r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"b0b1bee77f3193bd337ded006dace3e4a287d571250f957cf6c595348c7b326d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"8eab1b097f117f5fa387ae53997393440b33b00e85edf8ebaaec122145cf76b9","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001780ce0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b2v8ts8xj59w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-b2v8ts8xj59w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001780ce0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b2v8ts8xj59w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-b2v8ts8xj59w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"b6f76c09b1cce4e204168b9a9910280f8395b20c9334f4813a2e73e90dce7a26","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"e56a54c25506fef4fd4c8605a192ca2458a0dba96b5fecb9ab73009654f3cea2","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e02ca0\u003e: \n failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-7cp2xxm744zd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-7cp2xxm744zd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e02ca0\u003e: \n failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-7cp2xxm744zd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-7cp2xxm744zd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"b9c4a27aa6726fb8bc46395e3ca15d2c32caa65204aad3f76ddd63497052260b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"b5ec5970e9c7614d527f5257f02b1a83f847618cea746349aa1317c1475e49f9","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b6e7c0\u003e: \n failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-sh2zd6mpft2k\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-sh2zd6mpft2k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b6e7c0\u003e: \n failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-sh2zd6mpft2k\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-sh2zd6mpft2k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"baadc7a8b1a5b8ea06203338bd405879981dedb5619d7b2ff95de3884e726acc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"8d6a29106f6d7f3260c9698ad13eebf0c73b422c9bdeb8162463a6c655d9d521","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00163f5c0\u003e: \n failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-dnkpdfkrzkfr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-dnkpdfkrzkfr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00163f5c0\u003e: \n failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-dnkpdfkrzkfr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-dnkpdfkrzkfr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"bf05195f1482a49d2f1a18a98aaa18775204d1a5397fa24b5dc685025091fc4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"e62df2a6c20e4789165daee30b92b9ff0d162ba6cc3b030a9068c6492677d781","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fcef00\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-nvddkl6rpvcr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-nvddkl6rpvcr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fcef00\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-nvddkl6rpvcr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-nvddkl6rpvcr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"ca55b7c3d6708537e279fe34654ac7eda8bf838e3ec47bdfec3f937afe454c62","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"f4de3b4a6d6871ebd4e70e0b4bd0fa3064ac875b32d2edd0ec307ea339df236b","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-bn5fww\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d9b800\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-bn5fww: failed starting cluster creation \"cp-ystream-upgrade-5-0-bn5fww\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-nphnd2d8z9tk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-nphnd2d8z9tk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-bn5fww\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-bn5fww\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d9b800\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-bn5fww: failed starting cluster creation \"cp-ystream-upgrade-5-0-bn5fww\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-nphnd2d8z9tk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-nphnd2d8z9tk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-bn5fww\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"cba6285e86f9908b23bb62622687bb59f25299bb9c9963dfef82da25f9970ddc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"fd1596bb61848aaa7a93806f7b61359b36fdd12bacf8c47ddff014580b73c416","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-bnc5gf\" with version 5.0.0-rc.2 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00083ee20\u003e: \n failed to create HCP cluster cluster-zstream-5-0-bnc5gf: failed starting cluster creation \"cluster-zstream-5-0-bnc5gf\" in resourcegroup=\"rg-zstream-upgrade-5-0-spmmbkw6q2j9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-spmmbkw6q2j9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-bnc5gf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-bnc5gf\" with version 5.0.0-rc.2 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00083ee20\u003e: \n failed to create HCP cluster cluster-zstream-5-0-bnc5gf: failed starting cluster creation \"cluster-zstream-5-0-bnc5gf\" in resourcegroup=\"rg-zstream-upgrade-5-0-spmmbkw6q2j9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-spmmbkw6q2j9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-bnc5gf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"ccdd9899201a6c2c277ded2f2677ee28fc77c2c832895d0de02f89632d83bbaf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"b97260f6b219bb2a5cecd3306c8357f6c318ddfbf92f98f909285fe3dad5b88b","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001220680\u003e: \n failed to create HCP cluster v20261001: failed starting cluster creation \"v20261001\" in resourcegroup=\"v20261001preview-pwf9pxbkvk9w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-pwf9pxbkvk9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20261001\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001220680\u003e: \n failed to create HCP cluster v20261001: failed starting cluster creation \"v20261001\" in resourcegroup=\"v20261001preview-pwf9pxbkvk9w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-pwf9pxbkvk9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20261001\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"d539a7cefa372300c6eae73aa89214b344bd8e6b8b7acf4fa21367bae028cad3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"08024cf849a325cbc5a336512e48936583c3f85cb4a3a2c148f103673d901cfe","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-25fcqw\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f12e80\u003e: \n failed to create HCP cluster np-noedge-25fcqw: failed starting cluster creation \"np-noedge-25fcqw\" in resourcegroup=\"rg-np-noedge-25fcqw-mkk44mbtzfcl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-25fcqw-mkk44mbtzfcl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-25fcqw\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-25fcqw\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f12e80\u003e: \n failed to create HCP cluster np-noedge-25fcqw: failed starting cluster creation \"np-noedge-25fcqw\" in resourcegroup=\"rg-np-noedge-25fcqw-mkk44mbtzfcl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-25fcqw-mkk44mbtzfcl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-25fcqw\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"d8bb720a59ae1816c3f2643d271584289b5f1e892fedd3ba1724a7fb8873761f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"996575218bcbe29d31c4ada2dd5e23243f75e74c9955fbef49e8df81a1d3822d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-ttflt6t5z8w7/cluster-candidate-4-23-kznbgd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d89720\u003e: \n failed to create HCP cluster cluster-candidate-4-23-kznbgd: failed starting cluster creation \"cluster-candidate-4-23-kznbgd\" in resourcegroup=\"rg-candidate-4-23-ttflt6t5z8w7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-ttflt6t5z8w7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-23-kznbgd\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-ttflt6t5z8w7/cluster-candidate-4-23-kznbgd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d89720\u003e: \n failed to create HCP cluster cluster-candidate-4-23-kznbgd: failed starting cluster creation \"cluster-candidate-4-23-kznbgd\" in resourcegroup=\"rg-candidate-4-23-ttflt6t5z8w7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-ttflt6t5z8w7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-23-kznbgd\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"dd9b83e0afcc324c49d23b52d3ca1125eafb9742ec3b2c339af040d7db367ff2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"708386bc40e3cb953a5fa1b68320e189ab8f616758c7e5e8216987b789b6a323","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ab2980\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-82nxjntf8mtx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-82nxjntf8mtx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ab2980\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-82nxjntf8mtx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-82nxjntf8mtx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"e70baded9ebe43e8d9c67fbcaabf2005a43b1edf7d4d2835a8f8c657405b77c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"bf64f485c1cc87388187d9c3b48f24e4866df71b94091b1f848430665c4c6977","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:101]: failed to begin creating HCP cluster \"admin-cred-lifecycle-n8fmrx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001578f40\u003e: \n failed starting cluster creation \"admin-cred-lifecycle-n8fmrx\" in resourcegroup=\"admin-credential-lifecycle-test-g8tk97spx6sk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-g8tk97spx6sk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-n8fmrx\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:101]: failed to begin creating HCP cluster \"admin-cred-lifecycle-n8fmrx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001578f40\u003e: \n failed starting cluster creation \"admin-cred-lifecycle-n8fmrx\" in resourcegroup=\"admin-credential-lifecycle-test-g8tk97spx6sk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-g8tk97spx6sk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-n8fmrx\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"ea9b36a43c1f7b164846681e11b59a87244b5ac07f339410c72a84320cae6f39","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"9baa846c847b6340711d575cf15d5a87dff938dfbaeea48c84709a07d2be7a00","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa6520\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-xzp28kftfsgl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xzp28kftfsgl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa6520\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-xzp28kftfsgl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xzp28kftfsgl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"efd87dfd714cb4ebc016cbc591566fdba3f236fec7762b745dc4f3c4927d7c45","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"1efd0a59df6c4ca49a656f7bba9764b2b191d6d3446eb06a4a269c70541d699f","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-5c9stf at version 4.22.15\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000816b20\u003e: \n failed to create HCP cluster np-skip-minor-5c9stf: failed starting cluster creation \"np-skip-minor-5c9stf\" in resourcegroup=\"rg-np-skip-minor-5c9stf-zgbfk5cxflhd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-5c9stf-zgbfk5cxflhd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-5c9stf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-5c9stf at version 4.22.15\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000816b20\u003e: \n failed to create HCP cluster np-skip-minor-5c9stf: failed starting cluster creation \"np-skip-minor-5c9stf\" in resourcegroup=\"rg-np-skip-minor-5c9stf-zgbfk5cxflhd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-5c9stf-zgbfk5cxflhd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-5c9stf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"f5a54d8d4976ac80d8409abaa965c390adde0a3770f0b89eb96d6116fb7b83d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"1d9102a2f82649aa9433fd7ca07f7f8568768752d9d21e09bf205c33862544e5","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0018ece40\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-cwk7z6fq2hf6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0018ece40\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-cwk7z6fq2hf6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}},{"row_id":"fa3bc9bdd63b41b9a5109cbb14d5a5d88c413d562873d90fbeaac9c07774c8c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","failed_at":"e2e","signature_id":"30e39a8546827489b79b43d722c71489114e6cee327dd845c522ff10e2a9e7f1","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000119340\u003e: \n failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-m4g9t2jskz2r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-m4g9t2jskz2r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000119340\u003e: \n failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-m4g9t2jskz2r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-m4g9t2jskz2r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: LocationNotAvailableForResourceType\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LocationNotAvailableForResourceType\",\n \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType"}}],"failure_pattern_summary":{"signature_count":59,"failure_pattern_ids":["68cf65637733da6389b66f0d98a7fd646026fde9e15cc697ce29959bae80ec43","7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae"],"matched_occurrences":59,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T13:55:50Z","started_at":"2026-09-24T13:55:50Z","completed_at":"2026-09-24T15:24:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T15:41:39.395707009Z","timing_metadata_checked_at":"2026-09-24T15:41:39.395707009Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T15:41:39.401572989Z","region_metadata_checked_at":"2026-09-24T15:41:39.401572989Z"},"failed_at":["e2e"],"failed_test_count":55,"occurrences":[{"row_id":"03753bdf1a5fc12f3ec71343b841b3d403e4b48de8b277f2a46e99c6490581ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"07f72b351d9885024483bcdda053ed042e3ef275bc57e4fe920a64d613ecaef3","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-q9n7xf\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015a82e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-q9n7xf: failed waiting for cluster=\"cp-ystream-upgrade-5-0-q9n7xf\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-mbh76rk9sprx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e50bc50d-ed3a-44e5-99bf-5d82436a9665\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-q9n7xf\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015a82e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-q9n7xf: failed waiting for cluster=\"cp-ystream-upgrade-5-0-q9n7xf\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-mbh76rk9sprx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e50bc50d-ed3a-44e5-99bf-5d82436a9665\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"07129a8d0de30e9efea8f2eebb102c7ba8f39ad23ef748042544fead8bd4c551","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"ff83937b404baac989bbd19b80b0cf998ad2d8f2250a11efd067cd3656f1c535","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001106580\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-rjr22k9ck9q9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e00f99a2-ee29-44a8-b972-faed7dd9107f\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001106580\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-rjr22k9ck9q9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e00f99a2-ee29-44a8-b972-faed7dd9107f\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"096adc2fd4455b90dd7053ed820696432cea3e87956940ea49122e961291c2c3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"92f971cf89077bcf6f7c95c75f6548c5f63e238aa46a1aae576b8a17cd5e41f3","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c9afe0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-gnrv6kblsb4v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2ff2bc98-36d7-4657-b844-1f907e6770ba\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c9afe0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-gnrv6kblsb4v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2ff2bc98-36d7-4657-b844-1f907e6770ba\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"0c791ddf31878f68b8e155248c3dc6cd58b4203e86fc5fa9d86d1f5caa005c83","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"4c79d1369b373f888a3b45bc9980f8607e90b91cf48ef5a98e4523e05cd34677","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011ba1c0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-gmkdjq6v4l8d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/87a6922d-74fc-4481-b171-9e921e47725f\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011ba1c0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-gmkdjq6v4l8d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/87a6922d-74fc-4481-b171-9e921e47725f\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"0eec52b3f2d5e5f44deddbfc4766d6ce4526bdeb4b47517e03579fb59caae998","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"02065b62056fdb1be37e1f143fa9e3989f4c00365df7604d5fc5f67b38deeb74","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-p49qttrgxjfs/cluster-candidate-4-23-h8xv9w should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a0e9e0\u003e: \n failed to create HCP cluster cluster-candidate-4-23-h8xv9w: failed waiting for cluster=\"cluster-candidate-4-23-h8xv9w\" in resourcegroup=\"rg-candidate-4-23-p49qttrgxjfs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/775da561-b2d7-4a7a-bbc2-e2713e469817\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-p49qttrgxjfs/cluster-candidate-4-23-h8xv9w should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a0e9e0\u003e: \n failed to create HCP cluster cluster-candidate-4-23-h8xv9w: failed waiting for cluster=\"cluster-candidate-4-23-h8xv9w\" in resourcegroup=\"rg-candidate-4-23-p49qttrgxjfs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/775da561-b2d7-4a7a-bbc2-e2713e469817\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1989d01911906aebaaaf195182805282f924fc25d37ec7b593dab8c568c2f4c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"da45a36375afba2b38843fbb63997abffdb70fe444c87c2a54f106bc6f210096","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aa81e0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-wfnsr7qsscm2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/26a20c52-fa37-4aba-b728-744aade6e61d\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aa81e0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-wfnsr7qsscm2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/26a20c52-fa37-4aba-b728-744aade6e61d\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1cb9c19c816558986ce50657a3e58592d922462554a1135f5fe977490c391275","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"25a79014eed6d04b3f486b4bd113cebd8ddda98d4daec17f489c251260e84d81","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f90380\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fz5ccggcv7ng\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/da969d58-8ed2-4cb8-98b5-5e49f25760a7\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f90380\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fz5ccggcv7ng\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/da969d58-8ed2-4cb8-98b5-5e49f25760a7\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1e852a7b252a064a5a20e9d38b8b230b30c64cb3eb4b256d68349a15936d9e78","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"6551a9274a31031d7c246caf9f38b3803753d5519917aa5228c3ef2caa0fdd99","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000af4360\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-fgdbfv79mc7d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5959ed79-fe6d-4e6e-9ad6-d8747b628ab2\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000af4360\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-fgdbfv79mc7d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5959ed79-fe6d-4e6e-9ad6-d8747b628ab2\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"246507be5b82fde0ebf4a33db8ab95985761eb7904f6f23a726143119aac6693","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"bdf59d33ef572133b373eccc869cfcbe6143231f3e4511de63e7d2a518123c1f","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00178a640\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-8n2xcvsmnqjs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b55f35c3-ff27-4b24-b063-f8561b8ce85d\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00178a640\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-8n2xcvsmnqjs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b55f35c3-ff27-4b24-b063-f8561b8ce85d\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2a27ee1dc2872e4dff6deb6d15222bee6c462fb5933a1f8f9bd18c8089d15a6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"fc7b68991da4719c194ace05c0d2d561215021a3d370ce936079026d3369b37e","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000936680\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-mv29z56qcdwh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94b0888e-53d5-420f-92bb-9785a421357a\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000936680\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-mv29z56qcdwh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94b0888e-53d5-420f-92bb-9785a421357a\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4abf52532ea3b43ec2f70e63622c8adc7cbd852d7908697a9e9c9a1257fa5c50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"1cec2a79be4975a9653bfd8882733838f7e123313a78520fb3323f7cd2a476f5","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00097c6e0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-ntxx89l7lfdg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e64c071-46f0-4057-8137-99940245e456\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00097c6e0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-ntxx89l7lfdg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e64c071-46f0-4057-8137-99940245e456\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4cabde2629279571b24928fd3a497c9247434710e55a3c71e2039f18534a8661","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"c6bd5ae45fe994223d884c6fd019d9bbc980c09700d74c7c6ec6d11950bf3641","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00166f260\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00166f260\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4fa4c40409064de8edb56acf4d3f23396974c8c6743fbef5a7d376d37d518474","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"344978cc9fbf73459cd931f815705d7f83f04f19bb1486d43bed4ef863a846f0","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:91]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b5e720\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-vbqclhz48q7d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dac67df7-9a2f-4479-ad7e-7f7a442c996c\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:91]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b5e720\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-vbqclhz48q7d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dac67df7-9a2f-4479-ad7e-7f7a442c996c\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"51cf0cec96e750edc36be23b87a74cc20de19642559cbf44162ad86b2e8f932c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"08a625905fbc7734eab24c2da8e1f2f3c7e204001ec83f60fdbbd150f3f95d10","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00173a180\u003e: \n failed to create HCP cluster np-dg-minor-mw6fw8: failed waiting for cluster=\"np-dg-minor-mw6fw8\" in resourcegroup=\"rg-np-dg-minor-mw6fw8-fv7854tv6zx6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6e066b83-100f-40e1-8a40-6aa10f3d747e\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00173a180\u003e: \n failed to create HCP cluster np-dg-minor-mw6fw8: failed waiting for cluster=\"np-dg-minor-mw6fw8\" in resourcegroup=\"rg-np-dg-minor-mw6fw8-fv7854tv6zx6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6e066b83-100f-40e1-8a40-6aa10f3d747e\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"5407c04eaa9c03e37d9bd3e39cc19bc74ec43f292d886ebcbb036bdc29b08196","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"853ae8067a45c4c5af2c0b037c1dc51d90a61b2a9935e489015743fa89a40ce2","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000900660\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-whdhftj7qn5d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6471fa0f-d154-448c-8e08-84fa720e0e52\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000900660\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-whdhftj7qn5d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6471fa0f-d154-448c-8e08-84fa720e0e52\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"638d03d0bab4066f44bc6342f858e69932286c1ef8f1949f8e6c582192380a90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"967b4f667dc4e0c254b60f79b36f69e3e0cba4279cc5c924e4597a63565d1a7e","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:238]: failed to get admin REST config for HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b4c8c0\u003e: \n failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-kbtzww7244jn\" to finish getting creds: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/53365482-c4a1-4a54-9bc2-e53d7610a2ef\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:238]: failed to get admin REST config for HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b4c8c0\u003e: \n failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-kbtzww7244jn\" to finish getting creds: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/53365482-c4a1-4a54-9bc2-e53d7610a2ef\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"65f76bb82d9bf14c7a2e69de928a600a348aeb554e8502fcdeecfa5114937c15","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"2491fcd2eac3fe129ef5af4e2165dbf276bb34dd6d7b915bcd09f12fdd5df0f0","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000898dc0\u003e: \n failed to create HCP cluster np-downgrade-gpn9mw: failed waiting for cluster=\"np-downgrade-gpn9mw\" in resourcegroup=\"rg-np-downgrade-gpn9mw-r62s5hw6qpcx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6b48c831-1af9-4e6e-9083-5e0aafbca54b\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000898dc0\u003e: \n failed to create HCP cluster np-downgrade-gpn9mw: failed waiting for cluster=\"np-downgrade-gpn9mw\" in resourcegroup=\"rg-np-downgrade-gpn9mw-r62s5hw6qpcx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6b48c831-1af9-4e6e-9083-5e0aafbca54b\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"67954b08e33de2ec7b970104cf9f89e3b441f6bcdd20673d5e5d338934dca697","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"28f63e4ea877a5226e030abfc09e558054018cbba5fc549e1976e79199d95ace","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b07980\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-ct4q5w8z58wh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/44875744-19fa-48cd-9879-d39c75585e03\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b07980\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-ct4q5w8z58wh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/44875744-19fa-48cd-9879-d39c75585e03\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6ccf83e829818d785a14b4cadf6492a2fcd0ac8dbf92a3decc2753c8ff10b401","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"2496e95a86576d0de268af2d478e259fba5936be455d300c7e3b637d4c9c0436","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d0e420\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-9h28b7rm7p7q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe262be4-afb0-46ac-ae85-2dea20ec65da\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d0e420\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-9h28b7rm7p7q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe262be4-afb0-46ac-ae85-2dea20ec65da\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6f2603cc39f8cfe5ac5ab1c97f3fc52677097c1b8403d9cf845dcef35ce3a910","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"e4898a44c2d0dd7c9bad670935c424400fc52e32d0af7e2fff6aa931d5cdaf60","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e34960\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-9l5kkq8td965\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e0f9e14-a270-4413-931e-d3e3b11a1154\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e34960\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-9l5kkq8td965\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e0f9e14-a270-4413-931e-d3e3b11a1154\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"72a0faf923322ecb705821b1971c6a6a18936af143da8c0ce21f5264e00bb9c3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"a742fb03b3de74be1d0b58aad6a2950c30af0f017f641245d24c851cefe749af","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001800360\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-fbch8rjgjcdt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/30333c79-2349-4fe1-aefd-512a2d96ca07\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001800360\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-fbch8rjgjcdt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/30333c79-2349-4fe1-aefd-512a2d96ca07\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7ecdc23fa052278023c81595c437b1b69f2886947f5e02f24ced10bf3b4e1a28","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"af40d913adbdce8e6103e8528992e78d1b771a4afc5a2f9de4c2c5f4a5305fe0","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-ttqwsf\" with version 4.22.14 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001084400\u003e: \n failed to create HCP cluster cluster-zstream-4-22-ttqwsf: failed waiting for cluster=\"cluster-zstream-4-22-ttqwsf\" in resourcegroup=\"rg-zstream-upgrade-4-22-hswnfth76bl9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9aba5af8-d524-474f-8952-a84e13286973\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-ttqwsf\" with version 4.22.14 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001084400\u003e: \n failed to create HCP cluster cluster-zstream-4-22-ttqwsf: failed waiting for cluster=\"cluster-zstream-4-22-ttqwsf\" in resourcegroup=\"rg-zstream-upgrade-4-22-hswnfth76bl9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9aba5af8-d524-474f-8952-a84e13286973\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"803d5dbb6a9ec70117edc120d67b2192d5d52e5477478cbf3f335b6026d79463","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"c356e51b9b454f478643d1102b07e3cc98f9d26f83f8834c798c0a3d0e6db18e","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001036880\u003e: \n failed to create HCP cluster ea-list46zfdd: failed waiting for cluster=\"ea-list46zfdd\" in resourcegroup=\"ea-list-79xdwcv5tf4m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/74932894-78b3-4bea-aeaa-d0849c360355\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001036880\u003e: \n failed to create HCP cluster ea-list46zfdd: failed waiting for cluster=\"ea-list46zfdd\" in resourcegroup=\"ea-list-79xdwcv5tf4m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/74932894-78b3-4bea-aeaa-d0849c360355\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8847f604bb51cd2880033ccd56adf1de7c26df2ad2f45087dbb706c8d84fca3e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"7c07a109a7f8eeb6001f12064984f5de864be533ff4a294d3a61fa4068710abb","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d24160\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-xszsmtdmvz98\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6dc4fe03-f2a4-42a5-b56a-4bd6880a15b4\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d24160\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-xszsmtdmvz98\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6dc4fe03-f2a4-42a5-b56a-4bd6880a15b4\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"88aca871e3dafb85131fdf63de15560fb8036360c18238da01a1dee721e120cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"d1479bc8ab6f8ba7adcc3af3628e73e527d7fb7d97afcf0aa9f15d21d77cfba4","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-v6qfj7 with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015da7c0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-v6qfj7: failed waiting for cluster=\"np-version-upgrade-cluster-v6qfj7\" in resourcegroup=\"rg-np-version-upgrade-v6qfj7-skg6jrnxc5p6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c7f3b390-27b8-429d-b49e-f6681f383963\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-v6qfj7 with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015da7c0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-v6qfj7: failed waiting for cluster=\"np-version-upgrade-cluster-v6qfj7\" in resourcegroup=\"rg-np-version-upgrade-v6qfj7-skg6jrnxc5p6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c7f3b390-27b8-429d-b49e-f6681f383963\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8afd11f65779d71aacef8f5c55d533005a299602e1970a4eeb33dbcb5b511dc4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"6a1c116e54ff19de705618b83b20d85085d2d56b50e18f8214b3986e60f160cd","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000828440\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-d87x9gm5jhzj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3877314f-c3de-45b0-bd82-35e621bcf79f\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000828440\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-d87x9gm5jhzj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3877314f-c3de-45b0-bd82-35e621bcf79f\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"94a05c17f163c10453531a390d1bf3f900d277be6fa70eca58d906cf412c7556","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"720afddf7bab9bd3106da1f01488f44f6c8d6565ba8f839cfcc4eb64d06a9278","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef01e0\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-pprdvsqzxr2q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0ad9bd8b-8087-4a46-86ff-311840c0da2a\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef01e0\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-pprdvsqzxr2q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0ad9bd8b-8087-4a46-86ff-311840c0da2a\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"962d344c5ae3a39db561dea3ff470ea9ee818938e467eeb1130fd248e4d5915d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"e2176f9fd669f072305456d519992c756a61ee3a64949c662685d7e2dec03a2f","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a36a60\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-2ztbw8htnxgv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/38bfbf27-f7bb-4d37-9b0b-8c8ea498da96\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a36a60\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-2ztbw8htnxgv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/38bfbf27-f7bb-4d37-9b0b-8c8ea498da96\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"96c362818fe257b61eb9a16212289e83ae1db7c555ee7c5dde9e382a39908475","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"dca63c35bedaaf7995439199f45e3ba19c7e03db5fa96015f6372c55929ccc8d","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-jbbr7p\" with version 4.21.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018025a0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-jbbr7p: failed waiting for cluster=\"cluster-zstream-4-21-jbbr7p\" in resourcegroup=\"rg-zstream-upgrade-4-21-rxn6xbk849kh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5866f782-63d0-4727-b1f9-fbb6238a4cf4\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-jbbr7p\" with version 4.21.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018025a0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-jbbr7p: failed waiting for cluster=\"cluster-zstream-4-21-jbbr7p\" in resourcegroup=\"rg-zstream-upgrade-4-21-rxn6xbk849kh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5866f782-63d0-4727-b1f9-fbb6238a4cf4\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"97d8917bf24ca31dc117b50d00f42f8564c46cee497848139e05c1126a88807f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"eaed15a6a2f8b355906a22e9b7c33ce97a5593a10c5d7448cb923809e315d678","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-cz9xt5tdn72x/cluster-candidate-5-0-5qdbnm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00060b4e0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-5qdbnm: failed waiting for cluster=\"cluster-candidate-5-0-5qdbnm\" in resourcegroup=\"rg-candidate-5-0-cz9xt5tdn72x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/44f6089e-d9b1-4a58-b233-8bf4446e75a6\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-cz9xt5tdn72x/cluster-candidate-5-0-5qdbnm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00060b4e0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-5qdbnm: failed waiting for cluster=\"cluster-candidate-5-0-5qdbnm\" in resourcegroup=\"rg-candidate-5-0-cz9xt5tdn72x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/44f6089e-d9b1-4a58-b233-8bf4446e75a6\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"98df46a8e0afed4d23f6495375c1ce036697ca052bb51506ae3449870c3d3acb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"befcba4a92c1063abce85eaebfd5f49c658485ca00a56047126b556f822a3702","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd05e0\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-l8gvsztb94w8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31016ec8-3d37-4831-8008-f5f433d4a7e6\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd05e0\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-l8gvsztb94w8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31016ec8-3d37-4831-8008-f5f433d4a7e6\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"9f6fbdb6915fbf9b27504be58581ee4dbcaf5bf1c1831a4985365f6c8d4da2cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"3b2acae45c01d2462c31e39a66ceda7d89be3fb818bc996fd0c77eb9f0f34519","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-84tr9g at version 4.22.15\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00140a300\u003e: \n failed to create HCP cluster np-skip-minor-84tr9g: failed waiting for cluster=\"np-skip-minor-84tr9g\" in resourcegroup=\"rg-np-skip-minor-84tr9g-bf984wxm78rb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78280dad-f48c-47ab-96e5-a964252849a6\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-84tr9g at version 4.22.15\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00140a300\u003e: \n failed to create HCP cluster np-skip-minor-84tr9g: failed waiting for cluster=\"np-skip-minor-84tr9g\" in resourcegroup=\"rg-np-skip-minor-84tr9g-bf984wxm78rb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78280dad-f48c-47ab-96e5-a964252849a6\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a3b518d8970b8868ce866c3cf157fe016e64406ae2f53c2df4c2c5ed5c67b561","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"36eeb6a1e1b48cf8e706db53d0be8d91860b5935eb3db33b0ac89092c3fe1e98","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0017b88d0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8f140fe3-a6eb-4a04-8ee6-f53c88e39f17\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0017b88d0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8f140fe3-a6eb-4a04-8ee6-f53c88e39f17\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a400e71b9219c56f718750a190ec72f872c8f51ab1284153bff5834f5d04fad8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"4825f3fea6b1d5d7b75547790c7d34a8a40c3c76a119b4bc51a6e1c1b00426cd","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-8dk887\" with version 5.0.0-rc.2 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000adc8c0\u003e: \n failed to create HCP cluster cluster-zstream-5-0-8dk887: failed waiting for cluster=\"cluster-zstream-5-0-8dk887\" in resourcegroup=\"rg-zstream-upgrade-5-0-l9598lhfkdvh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/09132225-4087-467b-a92a-dc104be7a5ec\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-8dk887\" with version 5.0.0-rc.2 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000adc8c0\u003e: \n failed to create HCP cluster cluster-zstream-5-0-8dk887: failed waiting for cluster=\"cluster-zstream-5-0-8dk887\" in resourcegroup=\"rg-zstream-upgrade-5-0-l9598lhfkdvh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/09132225-4087-467b-a92a-dc104be7a5ec\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ab4948131e3f1f208b282f55d013516e80e0f0c0169d54b3fdc89439a69dd492","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"fa74be0ea5ab5dc482d53b08882b7452407715e934f79592c346c3604704b150","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000172620\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-lhkhbbx7mz6b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/470867e6-8d72-4c8c-bcba-f7fc908984b2\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000172620\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-lhkhbbx7mz6b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/470867e6-8d72-4c8c-bcba-f7fc908984b2\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"adcc8dd02173313f1ee5145d7733c66614697ccfb9bb035ba4a39695a471d61c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"a098cb5165f51c38edade056cc88516c0f0a3f03634689a03285045f95820440","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-9dncbv with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00063e6e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-9dncbv: failed waiting for cluster=\"np-version-upgrade-cluster-9dncbv\" in resourcegroup=\"rg-np-version-upgrade-9dncbv-xp64h6q8pgbx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b67dc797-8bcb-4368-a47e-f51ae12621d8\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-9dncbv with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00063e6e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-9dncbv: failed waiting for cluster=\"np-version-upgrade-cluster-9dncbv\" in resourcegroup=\"rg-np-version-upgrade-9dncbv-xp64h6q8pgbx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b67dc797-8bcb-4368-a47e-f51ae12621d8\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"af668ca2897349c7377945be1fa825ac6c4c22f4f2047be0e0bbaf0298664522","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"c70f585d6ebd874ebdf87f09a6f5335bbd8ffbbe2510b6afa7a3cc5775812ae0","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-d6llpw2w8455/cluster-candidate-4-20-vvt6h2 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f9a580\u003e: \n failed to create HCP cluster cluster-candidate-4-20-vvt6h2: failed waiting for cluster=\"cluster-candidate-4-20-vvt6h2\" in resourcegroup=\"rg-candidate-4-20-d6llpw2w8455\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7da8b990-fede-4a5f-94f3-9b16f5a9b2c7\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-d6llpw2w8455/cluster-candidate-4-20-vvt6h2 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f9a580\u003e: \n failed to create HCP cluster cluster-candidate-4-20-vvt6h2: failed waiting for cluster=\"cluster-candidate-4-20-vvt6h2\" in resourcegroup=\"rg-candidate-4-20-d6llpw2w8455\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7da8b990-fede-4a5f-94f3-9b16f5a9b2c7\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"bdd78cfdbef9e811b56ffbd30a8f5dee4e0e6f94cac16c7b0e4b54394be43fa0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"e1d895a054a411be98f3dd14706c4b6811158995094d211b7e4fe2cf75726156","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-pf9n2c\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016f22e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-pf9n2c: failed waiting for cluster=\"cp-ystream-upgrade-4-23-pf9n2c\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-9jksqp8f2hr5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1d5ba86-b04f-4228-8b57-27eb837e9c7f\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-pf9n2c\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016f22e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-pf9n2c: failed waiting for cluster=\"cp-ystream-upgrade-4-23-pf9n2c\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-9jksqp8f2hr5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1d5ba86-b04f-4228-8b57-27eb837e9c7f\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c2dc760f305412412993872fa5dd0a70071cd3fa9441cb4da2724f55c937c345","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"260a8c3ed58b404fe205841beee67492d697ec8ff77258801ee95054dcda2df3","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:190]: failed to request admin credential 1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fb79e0\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-rz6kqx\" in resourcegroup=\"admin-credential-lifecycle-test-zz2pjh67gwwb\" to finish getting creds: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2ba375f6-9fe0-49c8-b17c-7f0b96934061\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:190]: failed to request admin credential 1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fb79e0\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-rz6kqx\" in resourcegroup=\"admin-credential-lifecycle-test-zz2pjh67gwwb\" to finish getting creds: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2ba375f6-9fe0-49c8-b17c-7f0b96934061\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c984bd314b2fe7e9cb6bf5dec25c98df472a8ead3c8e420f1f99142b1b7bec25","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"f44007f9ca5ba215de70c688ccbfb0a29ff777eb5bd506af4f556c69185fea90","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006068a0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-d684r5cmn2cw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56b84e34-7609-4b5d-86d2-51c899622a53\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006068a0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-d684r5cmn2cw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56b84e34-7609-4b5d-86d2-51c899622a53\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c9d7ca4f55ece25e7409249988d5f182c4b585488ba2f944420f15cc7f3e5834","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"f0a12fd2d92b9a02183a192dbf54b969f015ef75889c772832ec9cf6d3be6b18","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-kf6fv7\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009a86a0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-kf6fv7: failed waiting for cluster=\"cp-ystream-upgrade-4-21-kf6fv7\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-nlzv5dsxfrrc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e284640-004d-4939-97cc-47ce133ee77d\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-kf6fv7\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009a86a0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-kf6fv7: failed waiting for cluster=\"cp-ystream-upgrade-4-21-kf6fv7\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-nlzv5dsxfrrc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e284640-004d-4939-97cc-47ce133ee77d\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ceaaf203c3f41fad7944a533e481c975bf72e09a53dc8c2e98c676951c9f001d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"fa4f63d6822358edacc49b065a20427c02d8e8a557b09adc67f44ddfbe510a96","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001110320\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-tz5pdqdgjmrw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5ba22792-42a3-4249-8a6d-1ccaf5e1029c\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001110320\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-tz5pdqdgjmrw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5ba22792-42a3-4249-8a6d-1ccaf5e1029c\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d0ac778501864bc8d8a4c2afcaa6c12ed4cc665b9f30d0d801b2c23a53face33","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"40a33c47440e8587f6d078af5f5c16c35ffdc39ca033c2eee6186595c1a87db3","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001170440\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-snl76l5kjj4q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2debef79-d8a4-4dbb-b867-f8cba67ed38d\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001170440\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-snl76l5kjj4q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2debef79-d8a4-4dbb-b867-f8cba67ed38d\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d7356bcd7d56036f9b98e49e098f9277ce9456866718c1fc186e5ae4ca330fe9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"1d4510859afcc6e6a951c71249a3c54a4d1b3907f437de943b3dbcb9f98099bd","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-2sbqkx\" with version 4.20.38 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eeea80\u003e: \n failed to create HCP cluster cluster-zstream-4-20-2sbqkx: failed waiting for cluster=\"cluster-zstream-4-20-2sbqkx\" in resourcegroup=\"rg-zstream-upgrade-4-20-tk6s9fw45dg4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/762eb000-3830-4b9d-92b9-24d469611d95\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-2sbqkx\" with version 4.20.38 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eeea80\u003e: \n failed to create HCP cluster cluster-zstream-4-20-2sbqkx: failed waiting for cluster=\"cluster-zstream-4-20-2sbqkx\" in resourcegroup=\"rg-zstream-upgrade-4-20-tk6s9fw45dg4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/762eb000-3830-4b9d-92b9-24d469611d95\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d76e6d9e1f7ae52977c5f0b61dcb9fb096e3e372a0160286cf98ae5cfea80d38","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"967f8bf89265c03055c4a80ded5359ab4d74d96b7b327c590c07ba6380770600","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000788340\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-khvsfbtbxq2c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/be2c01e2-4220-4646-bc9d-468d7099961c\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000788340\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-khvsfbtbxq2c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/be2c01e2-4220-4646-bc9d-468d7099961c\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d7a6c59850135ab186d823231a4ebebd156a6d110abf529bb68ac9a868be1b01","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"ac2713765a7efcac04a7835d38ada2e943f7084564106cb6c2933a048a1e3802","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a44400\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-pffhm2cq6f4q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/72ff852d-07ea-4cf7-8b3e-75e91ccd6844\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a44400\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-pffhm2cq6f4q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/72ff852d-07ea-4cf7-8b3e-75e91ccd6844\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d7aa554ee1a0289958812be6e73e720d3147e5776cf8d3aae0ef67880f17cb5a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"0886a5c6a659b6005a58f499f6c69484cfe98d4e6289c7f072c5c6904bf7a54b","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000994380\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2v5hnvkbp6nz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ac014577-36dc-4208-bce8-f0e66ebed0a8\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000994380\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2v5hnvkbp6nz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ac014577-36dc-4208-bce8-f0e66ebed0a8\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d935a96951ffda69e98362101de6a779830eb41108ac6ef53ac374a44ba4ee49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"c243f446d5b36205395c8a9d49707a8bdd732debf06f3c70362d814c54919a5e","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012b4300\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-noedge-vf56pz\" in resourcegroup=\"rg-np-noedge-vf56pz-nwsgmksht5fp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/998f1803-4553-4717-9f65-bbfa6f28ace6\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012b4300\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-noedge-vf56pz\" in resourcegroup=\"rg-np-noedge-vf56pz-nwsgmksht5fp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/998f1803-4553-4717-9f65-bbfa6f28ace6\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e125b62ae1fd5531082d9b1c59ac7c7ccdd61dc6444ea86e802e1bff4b5dc323","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"f0d90df73e0ad52f9f8a704c64e5d1c3c05332de051b7b56f846352299f4f2e4","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001290360\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-ghbftw2cv4rh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/66de4363-9d94-46ea-bfff-873cb3efa5b1\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001290360\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-ghbftw2cv4rh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/66de4363-9d94-46ea-bfff-873cb3efa5b1\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e398bcab48ce65c834eeeb966952d2c8323738820e846dc67764d22d52d35c0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"59208ccacb71632d7b6b4858b15d2c7fc131d39ca9231040ecd620b898b8bc2e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-pctpkcgcwr47/cluster-candidate-4-22-fqmxwl should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c2b560\u003e: \n failed to create HCP cluster cluster-candidate-4-22-fqmxwl: failed waiting for cluster=\"cluster-candidate-4-22-fqmxwl\" in resourcegroup=\"rg-candidate-4-22-pctpkcgcwr47\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3b87b683-d3ee-4c0c-a392-3e0b820cd2d6\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-pctpkcgcwr47/cluster-candidate-4-22-fqmxwl should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c2b560\u003e: \n failed to create HCP cluster cluster-candidate-4-22-fqmxwl: failed waiting for cluster=\"cluster-candidate-4-22-fqmxwl\" in resourcegroup=\"rg-candidate-4-22-pctpkcgcwr47\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3b87b683-d3ee-4c0c-a392-3e0b820cd2d6\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ebda63273cf163b0215c6aae9491dec0be59469dbe327f8bf36c86d07e7a34d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"60ecda7dfb77cb74a1b276ceee25e1a751452462d2eb558d731953e82366126d","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-74dgdp\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a36500\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-74dgdp: failed waiting for cluster=\"cp-ystream-upgrade-4-22-74dgdp\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-m9l42vdrlhj8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ec116fea-ca7e-4a3f-9d3f-5f3c8f61a81e\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-74dgdp\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a36500\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-74dgdp: failed waiting for cluster=\"cp-ystream-upgrade-4-22-74dgdp\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-m9l42vdrlhj8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ec116fea-ca7e-4a3f-9d3f-5f3c8f61a81e\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f0ac48956975396f9102bd4809fc7275d76bb3ba3b1c76331d7281256d33e4ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"4de0b73b423ac48667ae452b214274712219be45eefe629403a4f7ca9649d73b","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f06380\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-sgj22g9zp2jg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ed833655-04f4-4a8d-83cd-dfd1d5a5550d\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f06380\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-sgj22g9zp2jg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ed833655-04f4-4a8d-83cd-dfd1d5a5550d\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f3dbb6f49251be9cda715dc05199a0272927f502d904719b953ed2386c253701","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"0948e3f0cfdd80324d14616eaa43a0f4cb0a32ab76564cb78669d9d2dcf1f472","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011104a0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-t9ln8p8l2tq5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7b953e56-e747-4da4-a09c-0706792a8104\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011104a0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-t9ln8p8l2tq5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7b953e56-e747-4da4-a09c-0706792a8104\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f45839708d52522bc026c9d89e5bb2a1e209120b961609a841090a51cc398bd7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"6cc2e47633c842a292f67376ce88252879c7a32f0fa0c8b61c35f7e7b1a49ce3","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d744c0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-6lxzqp285qjw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4c99174c-b9a9-41b9-b961-8dea7b33a43a\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d744c0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-6lxzqp285qjw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4c99174c-b9a9-41b9-b961-8dea7b33a43a\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"fd8775ff4fd0b7c5bb2b569f661fd3ecdde5759e0f3b3cfcb0887aa3c79a71f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","failed_at":"e2e","signature_id":"5fa3292ca69ac2b8f8914d64f6c6e71c41533069457c029bc02a6af50ebe3d12","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-cjhdsrxcz2fx/cluster-candidate-4-21-ktgnhz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d5e480\u003e: \n failed to create HCP cluster cluster-candidate-4-21-ktgnhz: failed waiting for cluster=\"cluster-candidate-4-21-ktgnhz\" in resourcegroup=\"rg-candidate-4-21-cjhdsrxcz2fx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/db8d62fd-625d-44ba-891b-e4b1141e3aaf\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-cjhdsrxcz2fx/cluster-candidate-4-21-ktgnhz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d5e480\u003e: \n failed to create HCP cluster cluster-candidate-4-21-ktgnhz: failed waiting for cluster=\"cluster-candidate-4-21-ktgnhz\" in resourcegroup=\"rg-candidate-4-21-cjhdsrxcz2fx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/db8d62fd-625d-44ba-891b-e4b1141e3aaf\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}}],"failure_pattern_summary":{"signature_count":55,"failure_pattern_ids":["82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54"],"matched_occurrences":55,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T09:46:15Z","started_at":"2026-09-24T09:46:15Z","completed_at":"2026-09-24T10:08:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T10:16:50.063431225Z","timing_metadata_checked_at":"2026-09-24T10:16:50.063431225Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T10:16:50.066347915Z","region_metadata_checked_at":"2026-09-24T10:16:50.066347915Z"},"failed_at":["e2e"],"failed_test_count":58,"occurrences":[{"row_id":"0e6fc6681e8ea8ba9d5c47607952676bab6086401485b3f07c8446f6dccfcfa4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"86cd7d2264f1df006b68597c7229c5b754c8bda3b5ed006204597c09ea46185d","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eb4320\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-cwpk2drntd97\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/faf859b1-74ca-422c-b81f-40ff813f96a2\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eb4320\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-cwpk2drntd97\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/faf859b1-74ca-422c-b81f-40ff813f96a2\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1080c13a9849e0085b123f87ae85af674803914f173315cb432139e4eed4097c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"8d7cf3325e3f51dc9dd535c7d9b929725b10a46074af389b09950cf2fb1da226","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-fvtqgz\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018fe320\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-fvtqgz: failed waiting for cluster=\"cp-ystream-upgrade-4-21-fvtqgz\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-6r7mpcjl4xmc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e165024c-c364-4891-809d-d1b8ec4c0137\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-fvtqgz\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018fe320\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-fvtqgz: failed waiting for cluster=\"cp-ystream-upgrade-4-21-fvtqgz\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-6r7mpcjl4xmc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e165024c-c364-4891-809d-d1b8ec4c0137\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"12955a59ec002e405dbf3b5a69077d4fe4d4c729833a54b92d38a9002aa0e02b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"b69976e1243e6773fa8a60a6bf57877ebfa4d1090f0f4bf4996c7fd78024e054","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a863e0\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-xp5s75thlg74\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5a1c6cd7-0b9c-44ad-b5d5-b0ae7c231b30\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a863e0\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-xp5s75thlg74\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5a1c6cd7-0b9c-44ad-b5d5-b0ae7c231b30\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"14139fc962920010d30adbcc6def92833af92649b929c891148bb0919be66ca1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"b24096ca5ffba57fc98933a2488e339ea00b6584d3690b683f4ba35aeac04579","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-ld27nr with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d0e560\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-ld27nr: failed waiting for cluster=\"np-version-upgrade-cluster-ld27nr\" in resourcegroup=\"rg-np-version-upgrade-ld27nr-8f7lqzmgbkgh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/91dfcb8f-f729-443a-a281-0b106e15369f\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-ld27nr with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d0e560\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-ld27nr: failed waiting for cluster=\"np-version-upgrade-cluster-ld27nr\" in resourcegroup=\"rg-np-version-upgrade-ld27nr-8f7lqzmgbkgh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/91dfcb8f-f729-443a-a281-0b106e15369f\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1552e090d93e1c0f7ec846d4d7f3d0c2ee6c3efe054e078160a619a34af1075a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"0de87cbcf5c9877b7a4237a0f07ba9b9a2ebc8fcc8d43dd86eb431dfe3ce5bfe","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-d2xngq5ft4rm/cluster-candidate-5-0-p8cf4r should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001698320\u003e: \n failed to create HCP cluster cluster-candidate-5-0-p8cf4r: failed waiting for cluster=\"cluster-candidate-5-0-p8cf4r\" in resourcegroup=\"rg-candidate-5-0-d2xngq5ft4rm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c9af701d-bf05-4976-a460-12142920482e\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-d2xngq5ft4rm/cluster-candidate-5-0-p8cf4r should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001698320\u003e: \n failed to create HCP cluster cluster-candidate-5-0-p8cf4r: failed waiting for cluster=\"cluster-candidate-5-0-p8cf4r\" in resourcegroup=\"rg-candidate-5-0-d2xngq5ft4rm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c9af701d-bf05-4976-a460-12142920482e\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"15b1bd8c790542b788f8a61971304a9b668dd23981ea701b4318227a7b0976eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"c2585ec4226888f6f576e96e437f1b2775fc3ebb2fc2462aa71665f8f2c6de32","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000af78a0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-sph95f8lnpwg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1313ecbf-d0cc-4d2d-9b07-84a74a29744b\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000af78a0\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-sph95f8lnpwg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1313ecbf-d0cc-4d2d-9b07-84a74a29744b\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"229c5f80e8e602ed459d01aa78030191d02a7ffacbf98f9e67f9f805d7b05c2d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"b107cfaa0d2663af2e4815be747bec7cac7f76a98c014b790eba42d70dd84d23","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d08420\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-7n9t7cs46q4q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f9cbfc48-8e76-42d6-a1f1-a8774fadc2e3\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d08420\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-7n9t7cs46q4q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f9cbfc48-8e76-42d6-a1f1-a8774fadc2e3\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2350b0c9c36e7c17c146a893705bb750670fea0d2cc5b27aa22b5978891d24e5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"6fd0b24e06192fccaa00e3b1c3b94dc37d332c9b2cef1818ca04f0a13ee89195","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001932580\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mrlc55xjgcmk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9bb82f6c-adbb-43a6-962f-7137873d0f24\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001932580\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mrlc55xjgcmk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9bb82f6c-adbb-43a6-962f-7137873d0f24\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3b5e658a2e1543d637dd4cb0d3973971ec0697ebe1c3b7250832c2b647765616","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"3da6d511cfdfcc9a966ff17298c613d692f9d8b2b2ff41d8fa714b027d68841e","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000f21b90\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2e994b63-b5d1-4d53-9ad6-35d7d81b98ee\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000f21b90\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2e994b63-b5d1-4d53-9ad6-35d7d81b98ee\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3baf1a3c70105292c2b53c3f5a1071c2b819b94157c39ff846bb5593ae2296eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"5a72b9f3b17f460b30a9c29aa4a08aa0bffaf0211205623d232115f9e61c81d5","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001102140\u003e: \n failed to create HCP cluster np-dg-minor-9bjdkb: failed waiting for cluster=\"np-dg-minor-9bjdkb\" in resourcegroup=\"rg-np-dg-minor-9bjdkb-r76v7w6gdltx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/24bc8bbf-18f2-4cec-897d-e2520fc04755\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001102140\u003e: \n failed to create HCP cluster np-dg-minor-9bjdkb: failed waiting for cluster=\"np-dg-minor-9bjdkb\" in resourcegroup=\"rg-np-dg-minor-9bjdkb-r76v7w6gdltx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/24bc8bbf-18f2-4cec-897d-e2520fc04755\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"40a4ec8e0543b2839dac690945dd88f31ac4325a268564fa1dfd0a077afd74d7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"dc40facbe59320069107fd1a5edbe83f0018e18aca83e4dbb8e3245a4a984b31","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008a4480\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-4mwltxjn8kws\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/676e4bc8-7f06-4673-897b-29f113927a9f\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008a4480\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-4mwltxjn8kws\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/676e4bc8-7f06-4673-897b-29f113927a9f\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"436bd7457c518b12659371431948da9074df461893f487b7d7b779050222033e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"5bf9671d8b88883fbca8a3384f2c419073f4736cf8d87ba15eda4718b7e70b3a","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f042c0\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-8pm5zswdndlq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3a4ed24-0e21-47fa-a184-ad76c454abd9\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f042c0\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-8pm5zswdndlq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3a4ed24-0e21-47fa-a184-ad76c454abd9\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"510b838bc7a51e0986917ecac2e0d862c04b10764a52c7629a6b15bdfe705b6b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"bc73ce6f20fa67a9ec8483aefda2a2bd30ac78637b62659bb7edd381600f8d94","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012061a0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-qcrqtc27wlb2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a314c15b-4ea8-4cb8-a031-af4ed11745d9\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012061a0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-qcrqtc27wlb2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a314c15b-4ea8-4cb8-a031-af4ed11745d9\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"551fa10b5d1bc3619b2fb5e37403d61838884f8a5bfb7567bf14fcd23c88eedb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"5aeade5b036df850b438dd9d19f71214c80198c23e9c7a18159685d5d6127efb","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00056e5e0\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-pcgwd4zkwwvz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5dbc5fd5-8da6-4211-82d1-748fd3bbec20\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00056e5e0\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-pcgwd4zkwwvz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5dbc5fd5-8da6-4211-82d1-748fd3bbec20\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"5be901753b40f6a01b42a4804e5b4d2e474abbf085fab3ae7ce80b131569ea09","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"af55c67ce41cf5118af08bb66b3fe1ad5ba797a697ead874cb5b7babd4f9361e","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010183e0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-xl9vljxcqtsm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d795397f-d472-48b0-8abb-cee5783a4cab\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010183e0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-xl9vljxcqtsm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d795397f-d472-48b0-8abb-cee5783a4cab\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"5fb41806c9e03dbcb25a5ce264cbcc4b626118b84906c2ffdc9ca50cb219e2ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"46731606f59a9c0624f9396b197db65e833127d4c0d807797d4eb23355d2fef5","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-wx28cn\" with version 4.21.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007e25a0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-wx28cn: failed waiting for cluster=\"cluster-zstream-4-21-wx28cn\" in resourcegroup=\"rg-zstream-upgrade-4-21-989jzmj5vx85\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d6b54dcf-b16f-4851-8048-129f5e92a00d\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-wx28cn\" with version 4.21.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007e25a0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-wx28cn: failed waiting for cluster=\"cluster-zstream-4-21-wx28cn\" in resourcegroup=\"rg-zstream-upgrade-4-21-989jzmj5vx85\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d6b54dcf-b16f-4851-8048-129f5e92a00d\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"660ca8949d1a2ef3418868bf6fa7820357f8af59ab9b5381307c48323cc66e90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"a6d19019536f0ffdf7b8d102ecd1f15f3a7e0e6fbe23d1a367e4af2586c60298","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001086460\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-hhhh4fbv65rt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3aecc727-f2bc-4209-8da8-27f94954007a\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001086460\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-hhhh4fbv65rt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3aecc727-f2bc-4209-8da8-27f94954007a\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6646d36d1f428611c2534f00d856870d0b1984301b1020c6408fdf00edae6f79","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"f55bcf54c5d67b37e04f8a536611e03e87fb64780bc1a8305f99107bf6029933","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010ac200\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-8n6csl5lbht7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/428f6b41-8121-48a1-992f-9dbbf2d7af81\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010ac200\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-8n6csl5lbht7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/428f6b41-8121-48a1-992f-9dbbf2d7af81\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"66e68a9cfcc73deec67b1ab8c59eeb0a8016bea8fe921ea8ce1f0d0d1b5f96e5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"1f713ffd36c3707139930bf34758bcbcddc571154f3158c1cdc2ef4e6eb94f99","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00111cf60\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-58z5l9rhf4f6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a673e929-1d31-44e9-92e0-dad0c7c87a72\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00111cf60\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-58z5l9rhf4f6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a673e929-1d31-44e9-92e0-dad0c7c87a72\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"731d01a26ba6dc4e6b42560ff28fbfc864f7656117de40d12f048b75c66a841c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"78a9fc61dc31966c8f5382a6adefb305e57ed7ce933c7353322e5f5babeac4ae","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-kdmnj4\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006e8e80\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-kdmnj4: failed waiting for cluster=\"cp-ystream-upgrade-4-23-kdmnj4\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-97rzkflgq2nk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/adbdf657-33a9-407f-8747-1643c0105f4a\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-kdmnj4\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006e8e80\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-kdmnj4: failed waiting for cluster=\"cp-ystream-upgrade-4-23-kdmnj4\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-97rzkflgq2nk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/adbdf657-33a9-407f-8747-1643c0105f4a\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7ba923495d48bf299660abe1fcdd01b8e58abcaee9866267009e6eada6e03128","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"33a53d9e10e046f64807c0665d9a96bdf32e483709eca3a34cf7d6405c07a06a","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100a2c0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-2z4prgng29d4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e58606bc-443f-4bc1-b14a-e820a57ffeae\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100a2c0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-2z4prgng29d4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e58606bc-443f-4bc1-b14a-e820a57ffeae\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7d5b50564055860beea523b8455a3f077a9ddfafdc65ae372f76c99a222e8007","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"f38f22878ff4dc7e201880848f8c0291b74fc749dbdcde36131160cad489c060","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-rxg6m4\" with version 4.20.38 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f881a0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-rxg6m4: failed waiting for cluster=\"cluster-zstream-4-20-rxg6m4\" in resourcegroup=\"rg-zstream-upgrade-4-20-bbkphfsgk8l5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bacd2f24-2a1b-478d-a054-4475c06ee047\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-rxg6m4\" with version 4.20.38 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f881a0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-rxg6m4: failed waiting for cluster=\"cluster-zstream-4-20-rxg6m4\" in resourcegroup=\"rg-zstream-upgrade-4-20-bbkphfsgk8l5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bacd2f24-2a1b-478d-a054-4475c06ee047\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7e28b76145e3ecc9fae351dfc88e91bc71369eca6a8ed3adc2b5c149ebdafd52","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"8389b7d6027c2f391604e97713773076785e864f55a45ec1f88e9f79114dda44","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:91]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0029904a0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-9f25phpd7qnd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/926e4f43-e783-4a3d-8146-4a049b151d3d\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:91]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0029904a0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-9f25phpd7qnd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/926e4f43-e783-4a3d-8146-4a049b151d3d\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7f33e92420bffb82505a1a1dee1f2ca217a646046de28445bce951c5f43ea74d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"9276c54a5fbffccb202a50473f6fe048ba2a9c847ec78aa22b393009137283eb","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-jn62zfptdlfh/cluster-candidate-4-22-bhd4rp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e5c2e0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-bhd4rp: failed waiting for cluster=\"cluster-candidate-4-22-bhd4rp\" in resourcegroup=\"rg-candidate-4-22-jn62zfptdlfh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f434c010-1066-4c2d-b10d-e80bae8b14bb\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-jn62zfptdlfh/cluster-candidate-4-22-bhd4rp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e5c2e0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-bhd4rp: failed waiting for cluster=\"cluster-candidate-4-22-bhd4rp\" in resourcegroup=\"rg-candidate-4-22-jn62zfptdlfh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f434c010-1066-4c2d-b10d-e80bae8b14bb\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"80784bb60621f87a4744af2117ed54227ca0a1ce73affe25eddafe5ab72b3f03","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"6aedc299554b7d252bc79ff70f6fb948566ab1c2514ae26ec861e15fb35fabb5","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a034a0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-kj7lztprpdgp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2a7a23aa-7e52-4cb5-85bc-a2781b58b1db\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a034a0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-kj7lztprpdgp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2a7a23aa-7e52-4cb5-85bc-a2781b58b1db\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"82cb1222899ba7d21a0b1ed77d4f1e86cc83309ec69ffb1ade4e13051bb740f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"7c6822e43843c1ef4b10f112f34e70d1c8fbbbade14e202dbf3988631790bee2","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:238]: failed to get admin REST config for HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006c2ea0\u003e: \n failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-v7vgdzrnb8bv\" to finish getting creds: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bcb4f189-fe27-454c-928a-272dd865e652\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:238]: failed to get admin REST config for HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006c2ea0\u003e: \n failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-v7vgdzrnb8bv\" to finish getting creds: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bcb4f189-fe27-454c-928a-272dd865e652\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"86347c793f00989a4a650211e7f181cb40c98a6cbb1715f9a23f4802d6fb6da2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"dcd5e0fbecd4cda0aa2fa67497226a4350e6689b2cc3138bafbad6f78f1ff4e1","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:190]: failed to request admin credential 1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006212a0\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-4pv4j7\" in resourcegroup=\"admin-credential-lifecycle-test-4sn6n5rmm9wr\" to finish getting creds: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0db35508-1cd6-4c7b-8bdd-a380971142ae\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:190]: failed to request admin credential 1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006212a0\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-4pv4j7\" in resourcegroup=\"admin-credential-lifecycle-test-4sn6n5rmm9wr\" to finish getting creds: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0db35508-1cd6-4c7b-8bdd-a380971142ae\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"89052c50c71376223f5e66a4dd167bbd0b3a545be54695db01e2d71163194346","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"c28773c54c9830f7b9e29005190ee26d6777b293c6a190ba25bd70d0a8a59efb","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d44160\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-hn22tfmqv6xl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/be96828c-3724-4e43-acab-c552ce551ce2\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d44160\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-hn22tfmqv6xl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/be96828c-3724-4e43-acab-c552ce551ce2\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"89aa76979a03e178737bddd430af2bdb5520865857c633626c5bb281db739c48","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"43e7f1565c600241e1717dd21fe3ce831088fb47b5bcc65de9a0851b50e2a95d","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe65c0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-hkv2w5rmrwls\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4596f551-7934-4ecf-85de-65361c422156\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe65c0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-hkv2w5rmrwls\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4596f551-7934-4ecf-85de-65361c422156\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8bd2c81de7359817b97c4937812cd182691b1ed2cfab261ad067d6e4b8b6dd54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"dea6b577a82026d7021eb80143d812f0e84b8b2c556ecfee6f5490a8a7dcd1a3","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00056e520\u003e: \n failed to create HCP cluster ea-listr7gzhr: failed waiting for cluster=\"ea-listr7gzhr\" in resourcegroup=\"ea-list-q4f4j8wlsbgz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7b621f6b-d48c-483e-947d-297263d4c5b1\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00056e520\u003e: \n failed to create HCP cluster ea-listr7gzhr: failed waiting for cluster=\"ea-listr7gzhr\" in resourcegroup=\"ea-list-q4f4j8wlsbgz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7b621f6b-d48c-483e-947d-297263d4c5b1\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8d2e0478d3d76acda59ecd59609f14f36994c4ff063e90e48b348ecc9de032e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"6708cf6358c16ce63138d022c53fea4bcf335fd1b118e1a4b673daa90384d30b","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-hmhj8c\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018ee380\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-hmhj8c: failed waiting for cluster=\"cp-ystream-upgrade-4-22-hmhj8c\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4rhpk8x2q865\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6372ad7b-58eb-487c-8a40-fe3745dd5ed5\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-hmhj8c\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018ee380\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-hmhj8c: failed waiting for cluster=\"cp-ystream-upgrade-4-22-hmhj8c\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4rhpk8x2q865\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6372ad7b-58eb-487c-8a40-fe3745dd5ed5\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"902898b87d804daa6ea9ecf66c01cd54e521c8bc013b1ad81d5502a81d6920db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"1c03a99aa561093294de0b7f438653c656628e4496548c3d82bdad67cc2948ff","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-pzxm4gv78clx/cluster-candidate-4-23-glrb6j should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00121c300\u003e: \n failed to create HCP cluster cluster-candidate-4-23-glrb6j: failed waiting for cluster=\"cluster-candidate-4-23-glrb6j\" in resourcegroup=\"rg-candidate-4-23-pzxm4gv78clx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ef5a635-8738-4780-8a64-b986c2c5543e\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-pzxm4gv78clx/cluster-candidate-4-23-glrb6j should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00121c300\u003e: \n failed to create HCP cluster cluster-candidate-4-23-glrb6j: failed waiting for cluster=\"cluster-candidate-4-23-glrb6j\" in resourcegroup=\"rg-candidate-4-23-pzxm4gv78clx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ef5a635-8738-4780-8a64-b986c2c5543e\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"91637a9f7653b63e32757c5b4259dc118a74c3d27ed3cb94a9c2e66c77597839","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"fda77eca2b5ca7a3f170a1eb6c18547b563d0dfccae707a6948d12a1c6e551f9","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00188a400\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2mv5fcnfnwhx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e15b827e-4275-470e-95ae-1bf6fedafeff\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00188a400\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2mv5fcnfnwhx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e15b827e-4275-470e-95ae-1bf6fedafeff\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"9a3e9ba75095ebb38da78ee2fcc11b2bb1e7ae7280dd5e22f71c5ab54120fed5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"b3c2fd359c06e85b6c30642e7db5c4f3164f911439b26a9ebe32dbbc216aba1e","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c74580\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-9r6lld9b6rn6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f689573b-8f56-4906-b119-4990f4dda41c\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c74580\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-9r6lld9b6rn6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f689573b-8f56-4906-b119-4990f4dda41c\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"aba9eb24e11a754bc1aad5f593fb8229d6cffcc5942e43b33ebe4690c65449cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"ad63ed6b9b2818b36afda8be9a47960e810863d76780c0fffb31e4de99ffdc65","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-ngkdfq at version 4.22.15\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010fa140\u003e: \n failed to create HCP cluster np-skip-minor-ngkdfq: failed waiting for cluster=\"np-skip-minor-ngkdfq\" in resourcegroup=\"rg-np-skip-minor-ngkdfq-gtgs97cnfw88\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6eaf0c32-6b6d-4990-9635-378ab7eb25d0\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-ngkdfq at version 4.22.15\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010fa140\u003e: \n failed to create HCP cluster np-skip-minor-ngkdfq: failed waiting for cluster=\"np-skip-minor-ngkdfq\" in resourcegroup=\"rg-np-skip-minor-ngkdfq-gtgs97cnfw88\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6eaf0c32-6b6d-4990-9635-378ab7eb25d0\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ac1cd237d4b9bcdac8b33f0fb3717c540766aebea05c7aae4960ecae9bba9cf2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"146147e4c288d084c17ebda4302106fa26b576d46636bfe7aac69b5bd259555c","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe2360\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-6h86gvb8f299\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5e6a42c9-7d5d-41d2-8caf-6e8d40b86373\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe2360\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-6h86gvb8f299\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5e6a42c9-7d5d-41d2-8caf-6e8d40b86373\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"b2da348f12d12197ace6e5cfb074b13af8c413a07d36b28550e079cf859a3a1f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"6d40f168b82b944aab38b5963703fbbf90d9e0d6a85bc07a462e258b20489400","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00061e360\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-26ns9mv9l52c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b9141558-2e58-48ca-85ee-2663e0c2091c\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00061e360\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-26ns9mv9l52c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b9141558-2e58-48ca-85ee-2663e0c2091c\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"bdbe1df2e829cb24fe5fc227e4dc5707c3389ef0043d75fc6522e4b2c9829a13","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"ad0100265512dd64b07c8991367895c07931a266bb11587312fb99858cd39dd8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-fsffnj4lh882/cluster-candidate-4-20-zx8qdk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001338180\u003e: \n failed to create HCP cluster cluster-candidate-4-20-zx8qdk: failed waiting for cluster=\"cluster-candidate-4-20-zx8qdk\" in resourcegroup=\"rg-candidate-4-20-fsffnj4lh882\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/71315e35-48d6-4f78-b20e-dd7d8e2f3de8\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-fsffnj4lh882/cluster-candidate-4-20-zx8qdk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001338180\u003e: \n failed to create HCP cluster cluster-candidate-4-20-zx8qdk: failed waiting for cluster=\"cluster-candidate-4-20-zx8qdk\" in resourcegroup=\"rg-candidate-4-20-fsffnj4lh882\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/71315e35-48d6-4f78-b20e-dd7d8e2f3de8\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"bf79ddd980429b8367feca6a9f50fdf33291575e83f2ff16553640aab33cc480","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"e70b708fe9e564e7f0383d82d6551c4cd363a16cb4041d76a97df34265bce1bb","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-s9btnw\" with version 5.0.0-rc.2 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f16300\u003e: \n failed to create HCP cluster cluster-zstream-5-0-s9btnw: failed waiting for cluster=\"cluster-zstream-5-0-s9btnw\" in resourcegroup=\"rg-zstream-upgrade-5-0-rpfhb4xbm8lf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/510afec5-4713-4c20-89bb-fb8bc789e01b\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-s9btnw\" with version 5.0.0-rc.2 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f16300\u003e: \n failed to create HCP cluster cluster-zstream-5-0-s9btnw: failed waiting for cluster=\"cluster-zstream-5-0-s9btnw\" in resourcegroup=\"rg-zstream-upgrade-5-0-rpfhb4xbm8lf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/510afec5-4713-4c20-89bb-fb8bc789e01b\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c07906567d92db68f718fd31084dac868e31a607667635565cf43cf513f91bd0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"ec8594b18716573125d007b01c1d0d6dddb69ebe6a0ac57f7d8b0911bc05b4d1","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d84400\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-vdpv9pf7pqnl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68c5db19-fb03-4720-9089-dbd9b6721ef8\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d84400\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-vdpv9pf7pqnl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68c5db19-fb03-4720-9089-dbd9b6721ef8\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c2a877c0f1643343d558a7937815ccf0ac3fa434b2207f4c61252e3095cf64f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"6c26b8862baf61fb815a9bcc378adad747c42890538b5734c6866f9b99fd3e4b","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-nnsgh4 with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006e7860\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-nnsgh4: failed waiting for cluster=\"np-version-upgrade-cluster-nnsgh4\" in resourcegroup=\"rg-np-version-upgrade-nnsgh4-vkgvlwcxc8v6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c32359b-9c7a-4507-be1f-f8838c95a6d2\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-nnsgh4 with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006e7860\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-nnsgh4: failed waiting for cluster=\"np-version-upgrade-cluster-nnsgh4\" in resourcegroup=\"rg-np-version-upgrade-nnsgh4-vkgvlwcxc8v6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c32359b-9c7a-4507-be1f-f8838c95a6d2\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c3ff1e10aed15f94c3904e608102d12ce66f4fc5c33b45bb539d69b2747400e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"f750b8949c6048df3f06cc60730c23f17eed1205b46d7b1bef29f0143eec2ad1","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f60140\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-nkvvdh9b6rc2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bcf1777c-753d-4ce5-b89a-e090b60007bf\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f60140\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-nkvvdh9b6rc2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bcf1777c-753d-4ce5-b89a-e090b60007bf\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c470f902017f3e9c5475d9f2bd61181e10c82f55fcbe86ffbc983815c922f406","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"c32d7fdb1bc89858a63402558b43f56a1300afac08a052f62e1c545d91b7df40","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00160d1d0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00160d1d0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c75b17a08f1a0d368e633b58f81d9215b0378b33648c92bcaecbda34fa8c378a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"48577c445231793eaba83204969b16f58df261f371ff2fcff7713d00ec5ff487","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c28320\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-q8ckfq2kzzrm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b0c27bc-1671-49c6-b045-aa37b0328d63\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c28320\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-q8ckfq2kzzrm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b0c27bc-1671-49c6-b045-aa37b0328d63\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ccf8d7157f35bed213aa9d84a8e961ec8320cd97d443696f101874b1b0b919ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"60061ef754525bd1335a2775069174d5f92f29de7213ad5df21582aa86897763","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000894780\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-2dkfkf57f74b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b05c995f-8ed2-412c-802f-d23c105fccb1\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000894780\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-2dkfkf57f74b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b05c995f-8ed2-412c-802f-d23c105fccb1\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d14421da0cf7247514fac18279d3dbd21625c9ca17906d6681475d3007547cd6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"46f505820c3047e75eaa0fdee320e69bd3f3b2fb1616f3230268704c75a50fa8","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000af82e0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-9pdh5npqgsvn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/977a97de-e4df-498a-abd0-2e9b6ea6790f\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000af82e0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-9pdh5npqgsvn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/977a97de-e4df-498a-abd0-2e9b6ea6790f\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d67f8fd783aee3653073f4159acc42493edc24ffa5c343b858330173746e321d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"390f99fccf43a9fdffa950872ef3818be3f84574f56671796e24f00762856e39","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-59dq65\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ed8340\u003e: \n failed to create HCP cluster np-noedge-59dq65: failed waiting for cluster=\"np-noedge-59dq65\" in resourcegroup=\"rg-np-noedge-59dq65-lrcfd4c5bv9q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1b1ec3e-4489-4f04-9f0c-48725a46b17b\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-59dq65\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ed8340\u003e: \n failed to create HCP cluster np-noedge-59dq65: failed waiting for cluster=\"np-noedge-59dq65\" in resourcegroup=\"rg-np-noedge-59dq65-lrcfd4c5bv9q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1b1ec3e-4489-4f04-9f0c-48725a46b17b\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d6d74413b29991ccb6c6e57db44d1d0a31c286d933668918a80b2b25e7e37df5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"ff4468a0a3f2435884308ecd6b5b5e9484371fe0029dd446ac35d4ff6a062092","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-qqq6rbqpzr7p/cluster-candidate-4-21-crlmr8 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b6b160\u003e: \n failed to create HCP cluster cluster-candidate-4-21-crlmr8: failed waiting for cluster=\"cluster-candidate-4-21-crlmr8\" in resourcegroup=\"rg-candidate-4-21-qqq6rbqpzr7p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/471f9b00-7794-4e69-b8a4-2656470605e4\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-qqq6rbqpzr7p/cluster-candidate-4-21-crlmr8 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b6b160\u003e: \n failed to create HCP cluster cluster-candidate-4-21-crlmr8: failed waiting for cluster=\"cluster-candidate-4-21-crlmr8\" in resourcegroup=\"rg-candidate-4-21-qqq6rbqpzr7p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/471f9b00-7794-4e69-b8a4-2656470605e4\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"dd5831d6b1afd62933ff4967db9202b12b7fe5e0a8da7e6e5070dc590a39d1d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"7beda16600f383a39e0d3cc113f1193f43bcc9b23fc90352dae1caee67106339","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a064a0\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-k925b7qbr9lx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/43d663d3-fa0d-46cb-bb16-08ea9b2ebd51\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a064a0\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-k925b7qbr9lx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/43d663d3-fa0d-46cb-bb16-08ea9b2ebd51\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"edb6496261b6e8e68ee59397c94ad5d27b71ff904500bc3549575fbe2ea345bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"8ec6de64c81fb8d075d11e6ffaa54baf82b7ba35b1232f6b902d46f2c56ae52f","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001467840\u003e: \n failed to create HCP cluster np-downgrade-65v5nq: failed waiting for cluster=\"np-downgrade-65v5nq\" in resourcegroup=\"rg-np-downgrade-65v5nq-kdkvxpkx4x6m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7f93af87-9a30-4349-b34b-57c392bd5f61\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001467840\u003e: \n failed to create HCP cluster np-downgrade-65v5nq: failed waiting for cluster=\"np-downgrade-65v5nq\" in resourcegroup=\"rg-np-downgrade-65v5nq-kdkvxpkx4x6m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7f93af87-9a30-4349-b34b-57c392bd5f61\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f11abff6dc07070e2f90147daa9dbef61ab6dfe049ed9de1f57758fa313569e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"3acb3552748b9fd13e54ddeaf5b07bebbcfe389a5c337c06b8aca75747b6ddba","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b0a4e0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-fsj64wtrrx9r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/855b8314-4383-4b5e-b217-f13fcb1fba98\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b0a4e0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-fsj64wtrrx9r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/855b8314-4383-4b5e-b217-f13fcb1fba98\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f148c319ad68bec652213433eac7fb06d14b2c4fddb6c4125c52f4506728519f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"4586bf9af4491d16af0f6a47fa3ff73e6e59cc51a18f4f7bc5b6a0748206dfd0","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00076a2a0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-tbgmd5ksgg4m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b893d11a-1012-4cfc-b8d4-0c75c8612b93\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00076a2a0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-tbgmd5ksgg4m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b893d11a-1012-4cfc-b8d4-0c75c8612b93\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f7de6cda5c11bdd1bca70d459a82deff711784d3ac72f9b078b80013f576ce48","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"11b0056d5f9eb91a6078381a70f2fad5d7118409110b509dab70404db8f3a474","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001017520\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gvp7kwdrzpqg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cea47209-8be4-4454-b59a-b2d2d3a02020\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001017520\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gvp7kwdrzpqg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cea47209-8be4-4454-b59a-b2d2d3a02020\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f837acc42774ba00f3ad1a5e8d87643d736d0123d20d4540979ae7b9f92254c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"bf050428c36d99a435724d4cc497547088fd3c8b62f9ae651e02fb9c333b72cd","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b1e040\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-z5kkgmdx6d7k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffb337e3-ea22-456f-98e2-73505c08aba9\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b1e040\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-z5kkgmdx6d7k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffb337e3-ea22-456f-98e2-73505c08aba9\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"fdf4698fb22f3a66c7118ecb5738c9f18d7c550260b7da47c740d9ff910f3d90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"d173739cb5ea2347d8714d542da4da098196c8a0c7005c4787d2d2146a3dfab0","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-2xq2jm with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00168a2e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-2xq2jm: failed waiting for cluster=\"np-version-upgrade-cluster-2xq2jm\" in resourcegroup=\"rg-np-version-upgrade-2xq2jm-f4fntnkwsc7b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c6a594c7-ff04-411e-8f23-7c633cfb8f90\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-2xq2jm with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00168a2e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-2xq2jm: failed waiting for cluster=\"np-version-upgrade-cluster-2xq2jm\" in resourcegroup=\"rg-np-version-upgrade-2xq2jm-f4fntnkwsc7b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c6a594c7-ff04-411e-8f23-7c633cfb8f90\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"fe8aa28a78701d0dea33cc9613874f6f8b47e223f4bc006cd0848a4454adf5f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"a7962d5682a436d220883c1d5f6842915a2a574c423b72fe7acf314cec931af4","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-z7586h\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f4c380\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-z7586h: failed waiting for cluster=\"cp-ystream-upgrade-5-0-z7586h\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-njzfgn9tq5s7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8c645b4d-7ccc-488a-99d1-bbd87ec339bd\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-z7586h\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f4c380\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-z7586h: failed waiting for cluster=\"cp-ystream-upgrade-5-0-z7586h\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-njzfgn9tq5s7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8c645b4d-7ccc-488a-99d1-bbd87ec339bd\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ff4b4efdf543da81ee1c2c4bd52c21c3ae4f9be3944fe5543c6ec2d7a36ea03d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"f46adce6b1993d0da3f46389c99620d37df456802b56bd77fce4645c6ff1537e","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-2f8qzb\" with version 4.22.14 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c8e180\u003e: \n failed to create HCP cluster cluster-zstream-4-22-2f8qzb: failed waiting for cluster=\"cluster-zstream-4-22-2f8qzb\" in resourcegroup=\"rg-zstream-upgrade-4-22-gd9jz8tlz7sq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe969e12-fb4e-4c38-a0fc-bf741d1193ac\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-2f8qzb\" with version 4.22.14 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c8e180\u003e: \n failed to create HCP cluster cluster-zstream-4-22-2f8qzb: failed waiting for cluster=\"cluster-zstream-4-22-2f8qzb\" in resourcegroup=\"rg-zstream-upgrade-4-22-gd9jz8tlz7sq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe969e12-fb4e-4c38-a0fc-bf741d1193ac\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"fff2daacd616ef76988316715f28ec0da146590e7b643e6654912d057cf32112","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","failed_at":"e2e","signature_id":"19bb3f1582062678b47c57215798e7ed69eae547e6905ca14e129c211f2fd95f","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00114a7a0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-2b22rvngvs8d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ba73cb16-b51f-486a-8cf3-ad7d6cb3101b\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00114a7a0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-2b22rvngvs8d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ba73cb16-b51f-486a-8cf3-ad7d6cb3101b\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}}],"failure_pattern_summary":{"signature_count":58,"failure_pattern_ids":["82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54"],"matched_occurrences":58,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T08:37:33Z","started_at":"2026-09-24T08:37:33Z","completed_at":"2026-09-24T09:15:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T09:24:51.871298408Z","timing_metadata_checked_at":"2026-09-24T09:24:51.871298408Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T09:24:51.875088795Z","region_metadata_checked_at":"2026-09-24T09:24:51.875088795Z"},"failed_at":["e2e"],"failed_test_count":58,"occurrences":[{"row_id":"001519dcb3845656daca38e0df8aea203052c242d3b651cf569b96db28263fb8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"587a41d8cddc604041bb53b0856b1237afdb02c7190fbbfe6c6a0aad7d417448","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010ce1c0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-z2x9kzpfjpcs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0c43c069-d635-43e0-b928-4061190e3f39\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010ce1c0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-z2x9kzpfjpcs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0c43c069-d635-43e0-b928-4061190e3f39\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"056d981d62bc46727a35c1fbaa7e458bd5cc30233bb14772612552a739ada92a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"8f3b0599281e22f58c3defe5f0d147b488c1b6a0f16143880bf51ef7355d85d2","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000930ac0\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-vgqpfgqrdt6c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2773dd74-f105-4eec-8543-dcfca8209086\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000930ac0\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-vgqpfgqrdt6c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2773dd74-f105-4eec-8543-dcfca8209086\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"0e40e96c5dca2335a6e2052cad570c9d7fd41d74d8f703fc219ff8aa4c26be30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"e259ff9177d1f12706b36fd539e24cc7a728c76256da9b5c37b773690fcf29fa","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0009d70b0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0009d70b0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"17c8a1b9bd87ec7122e07b4d6430fff6f687b76e5c66b57ed1f0b568915b5e16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"29ca14863816cf66991a82f9b46c9729fedc201d00186088b1efce79233b37e5","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014f2120\u003e: \n failed to create HCP cluster np-dg-minor-ll5r5h: failed waiting for cluster=\"np-dg-minor-ll5r5h\" in resourcegroup=\"rg-np-dg-minor-ll5r5h-kxt2jpft7wlk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c9e8ad1c-f40a-41ea-a448-d45cea9434f2\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014f2120\u003e: \n failed to create HCP cluster np-dg-minor-ll5r5h: failed waiting for cluster=\"np-dg-minor-ll5r5h\" in resourcegroup=\"rg-np-dg-minor-ll5r5h-kxt2jpft7wlk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c9e8ad1c-f40a-41ea-a448-d45cea9434f2\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"191e2d5637ee3315b433da4e77fb10090d2d1ffd0423a7495bc42617dc0ddd9b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"be867ed293f062a9bd769325496bdaf3731f033dfbaf096811cf5f767ffb8479","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009211a0\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-9g2pvfqcgvqk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/08b80599-f178-48a6-8e63-9bffb24ac253\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009211a0\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-9g2pvfqcgvqk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/08b80599-f178-48a6-8e63-9bffb24ac253\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"1d8d880287b3e72667ce8255752f0e1f830d5a061a18491794a62c2e6e0fc27c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"a85e78a9cdf1147b1107f8d145242f52a53fb32fc15b96f67ddb5c9c38894042","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006c6260\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-lxfxstq9hdc6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6be43266-8deb-477a-985e-217212b3c9c3\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006c6260\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-lxfxstq9hdc6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6be43266-8deb-477a-985e-217212b3c9c3\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"207983a5a04de0db48f3ebe3c7c13cfc356df7fd55c7a4cc3bfe525f924239dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"8d03287f7ed3f0349f7714b3bb9f7d51bb8a6961412a9db10097515ba650bd89","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-7zwmn2\" with version 4.20.38 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001406160\u003e: \n failed to create HCP cluster cluster-zstream-4-20-7zwmn2: failed waiting for cluster=\"cluster-zstream-4-20-7zwmn2\" in resourcegroup=\"rg-zstream-upgrade-4-20-6tvfgzhnb9f7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dac5e483-8093-43ab-a249-0cfe987da88b\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-7zwmn2\" with version 4.20.38 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001406160\u003e: \n failed to create HCP cluster cluster-zstream-4-20-7zwmn2: failed waiting for cluster=\"cluster-zstream-4-20-7zwmn2\" in resourcegroup=\"rg-zstream-upgrade-4-20-6tvfgzhnb9f7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dac5e483-8093-43ab-a249-0cfe987da88b\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"20b60ad7d345c6c7e142b7dbc7330175a097ec7968bad69da869934664141c0c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"e9a3aecc6dfb1296c32f694be25ac887dd48af1f7379a8a0c02b86cd85efca8a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-gmqrwq5g864s/cluster-candidate-4-23-hvdmh7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b92ae0\u003e: \n failed to create HCP cluster cluster-candidate-4-23-hvdmh7: failed waiting for cluster=\"cluster-candidate-4-23-hvdmh7\" in resourcegroup=\"rg-candidate-4-23-gmqrwq5g864s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68096478-e4e8-46af-b3df-410513444ada\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-gmqrwq5g864s/cluster-candidate-4-23-hvdmh7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b92ae0\u003e: \n failed to create HCP cluster cluster-candidate-4-23-hvdmh7: failed waiting for cluster=\"cluster-candidate-4-23-hvdmh7\" in resourcegroup=\"rg-candidate-4-23-gmqrwq5g864s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68096478-e4e8-46af-b3df-410513444ada\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"273c3e546c8ade74428ad4b7632aeb55bbe20005c38c46be972252c1e81ec2ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"4f8bced1b86c8983e52fabef07413b07a835bc6c67a4fb7b1ec887b5f4487e37","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-wx86j9\" with version 4.21.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f88240\u003e: \n failed to create HCP cluster cluster-zstream-4-21-wx86j9: failed waiting for cluster=\"cluster-zstream-4-21-wx86j9\" in resourcegroup=\"rg-zstream-upgrade-4-21-84nwx9kjvq9s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/24167e87-3944-4069-af0b-84a754828b65\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-wx86j9\" with version 4.21.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f88240\u003e: \n failed to create HCP cluster cluster-zstream-4-21-wx86j9: failed waiting for cluster=\"cluster-zstream-4-21-wx86j9\" in resourcegroup=\"rg-zstream-upgrade-4-21-84nwx9kjvq9s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/24167e87-3944-4069-af0b-84a754828b65\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"280ef4245a4578b26168abce2129322a07a35de3db358fad5117d625681f2902","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"75547c088779acedcc0f688ec015777e98c9c620b93369dbad684de8c96d5068","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-htjlq8\" with version 4.22.14 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001654420\u003e: \n failed to create HCP cluster cluster-zstream-4-22-htjlq8: failed waiting for cluster=\"cluster-zstream-4-22-htjlq8\" in resourcegroup=\"rg-zstream-upgrade-4-22-9qbd8bqls6jc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0e4a0cf4-07d6-461f-a890-6f064bd6c922\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-htjlq8\" with version 4.22.14 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001654420\u003e: \n failed to create HCP cluster cluster-zstream-4-22-htjlq8: failed waiting for cluster=\"cluster-zstream-4-22-htjlq8\" in resourcegroup=\"rg-zstream-upgrade-4-22-9qbd8bqls6jc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0e4a0cf4-07d6-461f-a890-6f064bd6c922\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2b68e6c435516afe997e761cfecd2c8d30fccb414556bcdd2004236b220c4558","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"ee72d14d762ba3e9d0e27f0a7cef6b5bdcbf2c73ce4814de75d5572f0890fae6","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a32800\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-dfxwwmw5pfzj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1fb11f91-985b-4696-85d4-b6884c2c3515\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a32800\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-dfxwwmw5pfzj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1fb11f91-985b-4696-85d4-b6884c2c3515\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"32edd18e80b6f8d7f52f5b3ecdfdcda90d2981f12f2eb5662349fb495150958a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"42dab5794581704633e8583d39e306fe7eaa69761c0b3bdde3ae6b578beaad85","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006c2cc0\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-bthlczlxh8pn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/74364df9-e093-4ce7-8de4-4462f8abffb3\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006c2cc0\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-bthlczlxh8pn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/74364df9-e093-4ce7-8de4-4462f8abffb3\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"37793cf5600630e2676779cd50af95480f81078999f2186096d514656b3efde3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"d8cfcee194e8bef7aebd6061716b0c8a42cdcab9130e4a23056c9e64bf846076","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b0e2e0\u003e: \n failed to create HCP cluster np-downgrade-mf92ds: failed waiting for cluster=\"np-downgrade-mf92ds\" in resourcegroup=\"rg-np-downgrade-mf92ds-74h5g5s57m94\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/60adbefa-dfde-42a1-ab60-90a912d72889\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b0e2e0\u003e: \n failed to create HCP cluster np-downgrade-mf92ds: failed waiting for cluster=\"np-downgrade-mf92ds\" in resourcegroup=\"rg-np-downgrade-mf92ds-74h5g5s57m94\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/60adbefa-dfde-42a1-ab60-90a912d72889\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4592b69ff1f4238b3150cdf2d742b397ccaeccbf1c8c2195acdf3df1e19bd0ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"457829d398eaa8002f3a2d02b33f53ab433c3ee054e691177cd91348662c5085","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0026e43e0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-4fs4j62vswlx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c7c82017-f2c4-4311-ac0c-894ea597df58\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0026e43e0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-4fs4j62vswlx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c7c82017-f2c4-4311-ac0c-894ea597df58\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"482cd4be3b34fc7d4cccbd4f44bcbf8207dd1c182d8f933ad0323890de64ecf6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"91bfbb27fa9e016442e0811e84d9a5a5a7c3beeb9aefb272528d99cbbfde1693","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-bjzsqc\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00165c1a0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-bjzsqc: failed waiting for cluster=\"cp-ystream-upgrade-5-0-bjzsqc\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-scvm2wn84kfb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/86c9a4c9-79c6-426c-b678-3f5b50e1372f\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-bjzsqc\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00165c1a0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-bjzsqc: failed waiting for cluster=\"cp-ystream-upgrade-5-0-bjzsqc\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-scvm2wn84kfb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/86c9a4c9-79c6-426c-b678-3f5b50e1372f\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"487faab9d08e1a9a5cda7cb2604390ff9dc0fba4d31993dfeef809277c7097da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"e5cd62f25356431728e737caac9f39e8423781573072667b098d01ae01dd194b","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:238]: failed to get admin REST config for HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c00720\u003e: \n failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-2vgmjks25m4m\" to finish getting creds: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8b769151-6e53-4fdc-879e-257a7ef5809d\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:238]: failed to get admin REST config for HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c00720\u003e: \n failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-2vgmjks25m4m\" to finish getting creds: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8b769151-6e53-4fdc-879e-257a7ef5809d\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4a765cbc04ab3b3ee2eb92e900eb6805ef7a8dea976360902227ecc7e336c398","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"cf518dbc290a9e0538e3f3c154869ccf678a1dcbe5e6134efa2facaa72d7ed91","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:190]: failed to request admin credential 1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00073dde0\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-8lf47p\" in resourcegroup=\"admin-credential-lifecycle-test-2rs7chhwpc4n\" to finish getting creds: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/43c02385-5fa8-4618-85da-cf7b4f4d1da6\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:190]: failed to request admin credential 1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00073dde0\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-8lf47p\" in resourcegroup=\"admin-credential-lifecycle-test-2rs7chhwpc4n\" to finish getting creds: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/43c02385-5fa8-4618-85da-cf7b4f4d1da6\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4be79431e8473ee8dda6da954b22fea36ba8233c0edd8b9dda227422c5101ea1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"b9ad954e8151aa5b0826d0d74bf5d49508ba660587f9fe7ab54350a7e715d196","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010ae120\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-rm2gp8bzthz4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3fca39ef-cfc8-43b3-9b0f-46e3f6b5f13d\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010ae120\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-rm2gp8bzthz4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3fca39ef-cfc8-43b3-9b0f-46e3f6b5f13d\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4ea929e14d3535ecc58ac9c070b69cf22ce6360a1f833fbcc27586af40850af8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"e205db983efa4c7fe30a3098f99c7b28db976345b998b688245438d48a09e9bf","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-bmxs6w with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00052f260\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-bmxs6w: failed waiting for cluster=\"np-version-upgrade-cluster-bmxs6w\" in resourcegroup=\"rg-np-version-upgrade-bmxs6w-2hrwrsfnjsgz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/76f0a7b5-18a2-4186-9264-5605e6b3ba75\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-bmxs6w with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00052f260\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-bmxs6w: failed waiting for cluster=\"np-version-upgrade-cluster-bmxs6w\" in resourcegroup=\"rg-np-version-upgrade-bmxs6w-2hrwrsfnjsgz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/76f0a7b5-18a2-4186-9264-5605e6b3ba75\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4edf17763d1d582e33574e7d8524ca08b7f68e538d154a92a3c12c8767dca55f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"423808405226253bf2780a731584c5c00ca80e8ac8b89cbe298d4b9bfaa20ef2","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010f61c0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-2f5s4s5292qz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/43bed0b7-637d-4f9e-a76c-dd1c0759f849\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010f61c0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-2f5s4s5292qz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/43bed0b7-637d-4f9e-a76c-dd1c0759f849\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"5039cac81c9caf18b18c8635f47bd7f7911654247a2290b9e96d929f10a91b9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"14576de538e43a898ea0afaebbf5578a0ad679c60e4b538e2c28a0d08595fffb","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0017f2570\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fdb8d5ee-e534-4c8a-bab8-e124a2281894\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0017f2570\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fdb8d5ee-e534-4c8a-bab8-e124a2281894\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"548a02e6b826ce71614d428fecc49ed295dd0cdba8063f297d9a83d22a6a233f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"3c66cf27a6f188c03e99e0e620fe8a8673cd9acbed4dbb1db198db2a90216cd0","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010ea200\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2mz766hrmsh4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a7d6ebbd-21d5-4168-b9ef-6769c57a7383\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010ea200\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2mz766hrmsh4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a7d6ebbd-21d5-4168-b9ef-6769c57a7383\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"592bee348d193712d9e17cb5e9489361626df91e3a4d464ee1663f34073b3fc0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"95d33091b2e3ed0ead5eb1ceae0e4c3fd0d888e8ace45c0efcb0c25533a58f9e","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-lpjzpv\" with version 5.0.0-rc.2 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001122160\u003e: \n failed to create HCP cluster cluster-zstream-5-0-lpjzpv: failed waiting for cluster=\"cluster-zstream-5-0-lpjzpv\" in resourcegroup=\"rg-zstream-upgrade-5-0-gcnzx6nn895z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3356f23a-d69a-4d18-a2fa-975aa1dba2ae\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-lpjzpv\" with version 5.0.0-rc.2 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001122160\u003e: \n failed to create HCP cluster cluster-zstream-5-0-lpjzpv: failed waiting for cluster=\"cluster-zstream-5-0-lpjzpv\" in resourcegroup=\"rg-zstream-upgrade-5-0-gcnzx6nn895z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3356f23a-d69a-4d18-a2fa-975aa1dba2ae\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"5acca39c02f79ed07be61e84a95a69cddcb36f76fc46b35d0aed64d38cc37884","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"945c3b0b90dd70635719bf5c4663085b88eaf4bfe2ed27a8f72db49c7e4b63c5","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e98480\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-pf8nhhfwdldv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ac18bad4-a3a9-4bae-b590-c11b5fae6e54\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e98480\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-pf8nhhfwdldv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ac18bad4-a3a9-4bae-b590-c11b5fae6e54\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"60422ba52fb19d9dd7969bdd6ea67a61b8c82207ad544944e6d4020b4c86c908","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"ca41ca40c2a7ea09a51dc74552627e7f098ec66a21f33ac2a37277b8e785ba2c","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00159e140\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-5mpvwz2274ds\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dba4e3ec-d50c-4db8-a47e-128c7f0b8594\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00159e140\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-5mpvwz2274ds\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dba4e3ec-d50c-4db8-a47e-128c7f0b8594\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"612842d45b089f5b77f7d9012aec49efc234b4707d9ae3dbbe4d8209dafac8b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"0567435825b2f1f46e7c8c0c92ccdf5c233255559fbb4ecd19ff369114130a56","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008c3960\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-9bd2bz6h9p9p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df51608a-347c-4931-9425-91b0f221a8ed\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008c3960\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-9bd2bz6h9p9p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df51608a-347c-4931-9425-91b0f221a8ed\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"666c049bbf6b8cb6a17a79eeb78e944404bc00143b7e5e577b1ea7ce3746be82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"1ad44de4f84225657a0f5171167d0ef345449d251f077c5d71045f9eb0644d77","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002b1c3a0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-jnp7zdxxg8cw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e8153a6-1e58-434a-8ce9-d05c5a075d11\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002b1c3a0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-jnp7zdxxg8cw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e8153a6-1e58-434a-8ce9-d05c5a075d11\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6c9a4897691064d2dd8b94bd04be980c8839219d1d7ecda90f87ec7714956189","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"3f517ece69506cbfef521fda194e3fbb39f7991afddffc45f05e2fdb1b402cc9","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-dvvd9lz9qfrf/cluster-candidate-4-21-2ncnls should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a4e180\u003e: \n failed to create HCP cluster cluster-candidate-4-21-2ncnls: failed waiting for cluster=\"cluster-candidate-4-21-2ncnls\" in resourcegroup=\"rg-candidate-4-21-dvvd9lz9qfrf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5fd73cfc-f003-451a-83bf-31b74bcd067d\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-dvvd9lz9qfrf/cluster-candidate-4-21-2ncnls should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a4e180\u003e: \n failed to create HCP cluster cluster-candidate-4-21-2ncnls: failed waiting for cluster=\"cluster-candidate-4-21-2ncnls\" in resourcegroup=\"rg-candidate-4-21-dvvd9lz9qfrf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5fd73cfc-f003-451a-83bf-31b74bcd067d\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7022bbb64bc61d071e051522658da4c1f9f513003ca4c2fa32fa9865d6d70328","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"63c131654901b385f7d641263b5a6f2bdaa7a6105cd00a04562f380454869986","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000802d80\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-z2h4x96fwvfv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/46f2b135-92d2-48b7-b2e1-ed95e6c6a6d2\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000802d80\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-z2h4x96fwvfv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/46f2b135-92d2-48b7-b2e1-ed95e6c6a6d2\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"744187c2c958a9824f6c48d8a53b64bb591fc01ae233a9089abb54d68b82f754","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"8a7efe49c03007d1fdf090e4785ee39ae4747a5c5e30553be13dcc9aafd36922","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-jszlc6\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00180a340\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-jszlc6: failed waiting for cluster=\"cp-ystream-upgrade-4-21-jszlc6\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-9c5wxsfjpwnk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/48384837-d606-4bf7-adb9-e5ebfc01f203\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-jszlc6\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00180a340\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-jszlc6: failed waiting for cluster=\"cp-ystream-upgrade-4-21-jszlc6\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-9c5wxsfjpwnk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/48384837-d606-4bf7-adb9-e5ebfc01f203\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7d6f8a38df2bf3712380cf35cd55d387d335fc69e3c29961d174f2c5764b6bd2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"6eb87e754a0c8e7de25c379f966fd9286c4d705599dac33f311b2d71c14b420b","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c4c940\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-94jgsqlwdmnq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ddf1ab16-458d-4415-a7f9-065212e41e1e\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c4c940\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-94jgsqlwdmnq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ddf1ab16-458d-4415-a7f9-065212e41e1e\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"84350c43042b21218026fa5d8c3b4b567bf3fd92861a041126af27f9767fece0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"015a62d796a14aafc93ac8fd9fcd0192bc1f4daaf775afe7d5216b4fbed2d422","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011ea140\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-fbvzdt8brrln\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04444596-313d-4dc6-b1bb-e85239ee352a\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011ea140\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-fbvzdt8brrln\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04444596-313d-4dc6-b1bb-e85239ee352a\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8dce4afd0cbd04cca055238f0449ae3b3ccc064523b066ddf8657d17f118f117","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"fac67d5451426d0f28252a0fbe031705ee4850720c8d46822b1614d3f7cd79da","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-fj4pb7 with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001042700\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-fj4pb7: failed waiting for cluster=\"np-version-upgrade-cluster-fj4pb7\" in resourcegroup=\"rg-np-version-upgrade-fj4pb7-qmmpv2hmtwpx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/316360e5-e07a-44b2-aa4c-7e7deed05703\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-fj4pb7 with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001042700\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-fj4pb7: failed waiting for cluster=\"np-version-upgrade-cluster-fj4pb7\" in resourcegroup=\"rg-np-version-upgrade-fj4pb7-qmmpv2hmtwpx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/316360e5-e07a-44b2-aa4c-7e7deed05703\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"98b749ee5a33bbf036897707df3229af6732e3087448dc1bcbea592b02dc2f1c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"028ea5601862fa590d5e2201ae315bf0b1513b6769b08ed07c16e2de0ae99250","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011a4380\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-mgqjjwc44whb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ee9b81a3-da8d-4906-9d58-b1a3eca2efb8\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011a4380\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-mgqjjwc44whb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ee9b81a3-da8d-4906-9d58-b1a3eca2efb8\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"996c85339506afa4ec063033dd84e9896af32b9cad87602d0929204a6076c16a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"87764a3a1c887346ee5ef8611cc6a8377a04492e4975331324104e456976820e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-hxj8jzjsj4wx/cluster-candidate-4-22-f2c7ls should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c955a0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-f2c7ls: failed waiting for cluster=\"cluster-candidate-4-22-f2c7ls\" in resourcegroup=\"rg-candidate-4-22-hxj8jzjsj4wx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f8e9c5fd-c381-42f0-b97f-a20bab0ee86e\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-hxj8jzjsj4wx/cluster-candidate-4-22-f2c7ls should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c955a0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-f2c7ls: failed waiting for cluster=\"cluster-candidate-4-22-f2c7ls\" in resourcegroup=\"rg-candidate-4-22-hxj8jzjsj4wx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f8e9c5fd-c381-42f0-b97f-a20bab0ee86e\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a0e6de6c297c04214d95f6cb6b8f761e118423d8184744d9e4f865c6d67e7866","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"08753c85308ae68cb595376a3e6a02505c40bc1c3f231edf2037c7c84ce54390","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-7mrwf2\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d02460\u003e: \n failed to create HCP cluster np-noedge-7mrwf2: failed waiting for cluster=\"np-noedge-7mrwf2\" in resourcegroup=\"rg-np-noedge-7mrwf2-w9l84zwng9gf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0e081b02-f88b-42de-ba62-6fadaf13df49\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-7mrwf2\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d02460\u003e: \n failed to create HCP cluster np-noedge-7mrwf2: failed waiting for cluster=\"np-noedge-7mrwf2\" in resourcegroup=\"rg-np-noedge-7mrwf2-w9l84zwng9gf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0e081b02-f88b-42de-ba62-6fadaf13df49\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a26482c5080c8a4d560a74ad328bb78d58efa36fdfd8d00b41a1195cf5dab14b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"70fa408a82d01d5e0229300a03628a7b7c697abcb7a4b9d83bddeb0c17eeaedd","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00067aae0\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-wzq8hbkhk8st\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/20bc14f5-a9ed-4d5d-ae0e-e61acc5df8a3\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00067aae0\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-wzq8hbkhk8st\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/20bc14f5-a9ed-4d5d-ae0e-e61acc5df8a3\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a345c5688b34d5aa3f131f61689ab49de2f177b7fbef2c7620566b7e70fd807c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"48d0bbbeff9ebd85c0d242660d3ec069eb2a39489e5e738e5a0af642de48e47a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-9hqllf7rhbnn/cluster-candidate-4-20-69x5dp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009a0700\u003e: \n failed to create HCP cluster cluster-candidate-4-20-69x5dp: failed waiting for cluster=\"cluster-candidate-4-20-69x5dp\" in resourcegroup=\"rg-candidate-4-20-9hqllf7rhbnn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5614ff0a-a8db-40c4-8d86-43931a0311b8\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-9hqllf7rhbnn/cluster-candidate-4-20-69x5dp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009a0700\u003e: \n failed to create HCP cluster cluster-candidate-4-20-69x5dp: failed waiting for cluster=\"cluster-candidate-4-20-69x5dp\" in resourcegroup=\"rg-candidate-4-20-9hqllf7rhbnn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5614ff0a-a8db-40c4-8d86-43931a0311b8\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a5658282630cd070a34184d59820ddfcb9e06747bb27546b2c075123f80ab8c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"9c4d03f3479a5c70388413a70424168b5e2b17ad421c1bf6766436364b979d1a","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c60960\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gxgjqjvxxxj6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8ff8a326-d85b-4588-8d64-4668f5c77da2\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c60960\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gxgjqjvxxxj6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8ff8a326-d85b-4588-8d64-4668f5c77da2\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ab812370e55e3a19bc1c041fd67bc5d934e15d9a3b65034d46455aed14f75df4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"f9c44b3e72b1127fae5b53f6d7a080bc8f4bc6b26d9f291fb7dba9911f6078c6","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007fa460\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-dhzt66pct8mh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/192cb9af-f4dd-4dfb-b72d-621de0f96c43\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007fa460\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-dhzt66pct8mh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/192cb9af-f4dd-4dfb-b72d-621de0f96c43\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"aee175419f051618c4a7a2d7952b208fb178de8b281d3230d1592a2b14c9a4a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"de2fd51098d4fccf5d1bb093ad14ac97bee13c527a30b84d2038a84112fb873e","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-mh5q44\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009fa4e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-mh5q44: failed waiting for cluster=\"cp-ystream-upgrade-4-23-mh5q44\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-dzbbg7dkxq5q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c60713bb-3780-4f3f-b848-5be7053f3663\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-mh5q44\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009fa4e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-mh5q44: failed waiting for cluster=\"cp-ystream-upgrade-4-23-mh5q44\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-dzbbg7dkxq5q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c60713bb-3780-4f3f-b848-5be7053f3663\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"b9618a769be5725f38bfc988fc1803aecb3105dc44c27cc296d113d437c5c562","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"8d9b406fbbb8cea6732da0cd609f796ba30974cc05d728b0633597b50f8c0abc","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c12aa0\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-vhrfcsbl2kmh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7eaf31f9-fff5-41e2-ab4e-b85d8caa782e\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c12aa0\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-vhrfcsbl2kmh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7eaf31f9-fff5-41e2-ab4e-b85d8caa782e\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"bb656b3e1f228d0f56fcc325cf7bbb08fefccbbb3e07a19df50dfea748676f10","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"cbb84a2251ba6846685f78b090898c0a3ac4b5ed2f079675620d0a5b6d6c16e4","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ddc4a0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-tb9jrz2hwj8c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3601a9e-bdb8-49f6-a169-d2bab0ef2a88\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ddc4a0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-tb9jrz2hwj8c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3601a9e-bdb8-49f6-a169-d2bab0ef2a88\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"bf7d9792f46699058b1e25e0613ed7a47643989dbd988f3f869885f63681d261","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"5343a8025b83b6578ac81a81ccd0c2c8f82d4d10cd53595fab1fde76e7b1139c","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a88200\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-p8j4tmlkfz9s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/323e35c2-8229-47c9-9ac2-6f239975f95c\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a88200\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-p8j4tmlkfz9s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/323e35c2-8229-47c9-9ac2-6f239975f95c\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"bfaca98193c6fa2d661990b8fa8df16fad99287ee0f42f92adb81164aa3e42b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"6fe4ceadf290c9cd17620e6045688aae8219b7ba271ace3ac6aa8d59bd4dc942","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a96320\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-rpjktqwsql26\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/74e939b7-1005-49c4-8393-bc500b6f83a5\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a96320\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-rpjktqwsql26\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/74e939b7-1005-49c4-8393-bc500b6f83a5\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c36006fd4486d443980eda88533f5afa3e0738039ffe6a1f01b054e81aaa0724","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"0b322092770ddc32a8a6514dee884547809162ca71778488279ef0d524e10934","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a805e0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-bw84xtccbhp5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/220ee6e2-189a-404f-9a4c-ac05385f929c\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a805e0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-bw84xtccbhp5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/220ee6e2-189a-404f-9a4c-ac05385f929c\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c9bb16cb872592da670cacc37af9465449ffdf3afa58906400f566409bbce0ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"6a3887e150d79b2470eac482314e5825c603dbdf1cd847403c74e9971b98c5f9","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-jzwpzg\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cb41c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-jzwpzg: failed waiting for cluster=\"cp-ystream-upgrade-4-22-jzwpzg\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-x2c887tw67g8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ee53a009-5244-40be-88eb-ee6e096be5ce\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-jzwpzg\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cb41c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-jzwpzg: failed waiting for cluster=\"cp-ystream-upgrade-4-22-jzwpzg\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-x2c887tw67g8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ee53a009-5244-40be-88eb-ee6e096be5ce\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"cb639982ec02c511fee4d65e90de660ca11d6bba5a55944d61d32ceebf7328ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"f8f1d68ea61706af805ffd1d510750baa305613520982e0bba79e29979c78292","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-rhxnvd at version 4.22.15\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d133e0\u003e: \n failed to create HCP cluster np-skip-minor-rhxnvd: failed waiting for cluster=\"np-skip-minor-rhxnvd\" in resourcegroup=\"rg-np-skip-minor-rhxnvd-p58vwfz6fjfr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7b9fc5fd-52eb-415c-9b34-eb04c67db17f\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-rhxnvd at version 4.22.15\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d133e0\u003e: \n failed to create HCP cluster np-skip-minor-rhxnvd: failed waiting for cluster=\"np-skip-minor-rhxnvd\" in resourcegroup=\"rg-np-skip-minor-rhxnvd-p58vwfz6fjfr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7b9fc5fd-52eb-415c-9b34-eb04c67db17f\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"cd1d9420101ee5dad82c460a2568d87061770b5975ee371be57b9ab4a4da90ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"c37bda4304af7e53e58a3919e84b1477a3a46695576de8ff29d4640d43029591","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd0140\u003e: \n failed to create HCP cluster ea-listcwghrc: failed waiting for cluster=\"ea-listcwghrc\" in resourcegroup=\"ea-list-87dmsp5cvfdc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4f422e3a-fbeb-4f55-8dc5-72f74cefb46e\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd0140\u003e: \n failed to create HCP cluster ea-listcwghrc: failed waiting for cluster=\"ea-listcwghrc\" in resourcegroup=\"ea-list-87dmsp5cvfdc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4f422e3a-fbeb-4f55-8dc5-72f74cefb46e\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d0c2392e8474cc1cbf8714f5c94bb90eb58d806f98122ddb238b532149d5969c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"30cacbef53a9519a12502d8baa9dbf2e4ec518774dc9498d9b70b0e3f05f5dfb","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-t4fq96 with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001664140\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-t4fq96: failed waiting for cluster=\"np-version-upgrade-cluster-t4fq96\" in resourcegroup=\"rg-np-version-upgrade-t4fq96-sxs2pw2sbfpm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b629eadb-edb1-436e-b96a-1333d66e0129\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-t4fq96 with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001664140\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-t4fq96: failed waiting for cluster=\"np-version-upgrade-cluster-t4fq96\" in resourcegroup=\"rg-np-version-upgrade-t4fq96-sxs2pw2sbfpm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b629eadb-edb1-436e-b96a-1333d66e0129\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"dad0393fdd68b5257dce22444226cf0aecd802b5b9f6e607b999b6b9faf2f0ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"f8ae1811368ec12293281df96c6cee170ccc9b5c6f2f7f431f5e0b46e83d6d68","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fce340\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-8g8nztgsvfrg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8181c71c-8aa7-4bb1-9d0b-cda7ba17b489\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fce340\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-8g8nztgsvfrg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8181c71c-8aa7-4bb1-9d0b-cda7ba17b489\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ddbfc0d2c8a18a080f35b24913d3efe4e85b9f1d6b2a08baf06ed9955d1e097c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"98bfd3289c6b69608510ac8f99b14ab0bffa05824f201bcb73f80da461684588","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101c180\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-zj4pb6lz9fcr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9f0857bd-047e-4488-8e92-d32991104e91\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101c180\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-zj4pb6lz9fcr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9f0857bd-047e-4488-8e92-d32991104e91\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e4584522496a7853443d7f09255a456359377bcb50fbdb94f539387059b7457c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"6b345d5ab6014d402a2d652c0c5909b61c6a4242f0cc2ffe16076dc13b9b7b17","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:91]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ff8720\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-s5fm48wzgxnx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf1fb47c-c0e0-48ae-9e60-b5f0bb511539\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:91]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ff8720\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-s5fm48wzgxnx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf1fb47c-c0e0-48ae-9e60-b5f0bb511539\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e8a2f2e4da86474d3ffaa3da376f05129054da9803d4906b7ff253e562a3f256","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"abc370c7132d3dd0cad4c01b1df33504e2a5ee15cd985778c8ebeb08c7378217","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00017eb60\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-z5bwmmgwqzxc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc662588-7be0-4f38-a33c-686764d69d47\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00017eb60\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-z5bwmmgwqzxc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc662588-7be0-4f38-a33c-686764d69d47\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ee6b1aed4af9757d460fb0d1d15b4a55069261b0bf21af580d941a600b446dd6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"ce49b331dea25dcc6be08ddaf7d9a319f6c26e31fbd36c370bcef547c757521b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-qwjhw2znn64b/cluster-candidate-5-0-dwllgv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f1e1a0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-dwllgv: failed waiting for cluster=\"cluster-candidate-5-0-dwllgv\" in resourcegroup=\"rg-candidate-5-0-qwjhw2znn64b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8390f874-3988-4d1a-9f9a-a231f2da3eab\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-qwjhw2znn64b/cluster-candidate-5-0-dwllgv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f1e1a0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-dwllgv: failed waiting for cluster=\"cluster-candidate-5-0-dwllgv\" in resourcegroup=\"rg-candidate-5-0-qwjhw2znn64b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8390f874-3988-4d1a-9f9a-a231f2da3eab\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f375d682d9c10db4751371c56a4772db86a3ff935fe20167b957539ac89d8914","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"1088ef782bb08563b7f5eedd4e1fa2ad693b60b2a406cedcb3466c7edeefd7d2","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a34a20\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-n9xkt4t8wxvs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c9309b1d-f416-4aea-87a8-fc29e712ceb5\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a34a20\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-n9xkt4t8wxvs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c9309b1d-f416-4aea-87a8-fc29e712ceb5\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f5a453b7f5e2219a87df9840440efaa5b8655d149f4afc46e16e68b37dcbb6da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"195e663d42f52c6c17db4a06e43816ce2ca15b96d6fe36189b86067a43dc4c68","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001626600\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-sjczp869j2n7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/772f5453-23ef-49ef-9bdb-63e2dbe81c91\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001626600\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-sjczp869j2n7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/772f5453-23ef-49ef-9bdb-63e2dbe81c91\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ffd8c5cecb6d3074c06dcc1376775496e01fcded6ed4f3a0323a83526885f48c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","failed_at":"e2e","signature_id":"20d5cc742c1cee73f1b04cd10f6ea72323a6cb2809aa9f9667747252b4035ef5","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00103a700\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-jlwhkxsvk8nq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/867cd450-604e-41a5-b442-4f55b470d528\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00103a700\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-jlwhkxsvk8nq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/867cd450-604e-41a5-b442-4f55b470d528\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}}],"failure_pattern_summary":{"signature_count":58,"failure_pattern_ids":["82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54"],"matched_occurrences":58,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T05:07:03Z","started_at":"2026-09-24T05:07:03Z","completed_at":"2026-09-24T05:27:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T05:34:04.290595298Z","timing_metadata_checked_at":"2026-09-24T05:34:04.290595298Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T05:34:04.293416189Z","region_metadata_checked_at":"2026-09-24T05:34:04.293416189Z"},"failed_at":["e2e"],"failed_test_count":59,"occurrences":[{"row_id":"014053efc14cbf590e44b0dd3759620998888a937c425418992eab8e202d21c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"515b93d5e37ef059a66fe272396b5e973c12ddc43056fe961348a3e80afb8c97","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-22-k7zrmf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001091f40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-22-k7zrmf-vzcmmv\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-22-k7zrmf-vzcmmv/operationStatuses/08584113795721473635\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-l6ttmcjqchrr/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-guxmg5tcq4f4g\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-03/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-22-k7zrmf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001091f40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-22-k7zrmf-vzcmmv\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-22-k7zrmf-vzcmmv/operationStatuses/08584113795721473635\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-l6ttmcjqchrr/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-guxmg5tcq4f4g\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-03/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"0e09b52b9f93e6e44e2c22cc19ee1fbf7726cba848fe094324354567d903937c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"f78aa7804dffd9c91eb7a06ca983c1ab58ba6e852310a8faab7a08106948a7f0","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:80]: failed to create customer resources for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae2620\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cx-rg-hcp-cluster-tczj7p\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cx-rg-hcp-cluster-tczj7p/operationStatuses/08584113797417778439\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-11/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-pmnmz2hvttl5/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fmxs36wu5sow2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:80]: failed to create customer resources for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae2620\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cx-rg-hcp-cluster-tczj7p\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cx-rg-hcp-cluster-tczj7p/operationStatuses/08584113797417778439\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-11/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-pmnmz2hvttl5/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fmxs36wu5sow2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"110b517b7d8661c13ed1dff01163e2ffb072839437a51a4b0d8dff1c04b7e8b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"bcb1f165d9dbb4bacc1f9a57572c8fb656f3e7dfb80956cca46af347257d19e5","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:93]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00292a760\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-autoscale-cluster-6x8jr2\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-autoscale-cluster-6x8jr2/operationStatuses/08584113795062405694\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-20/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-rp7t4n8bpkjs/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ssrwqrnhyyabs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:93]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00292a760\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-autoscale-cluster-6x8jr2\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-autoscale-cluster-6x8jr2/operationStatuses/08584113795062405694\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-20/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-rp7t4n8bpkjs/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ssrwqrnhyyabs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"15befbd748f56467ee075f93c9e35acb66c0a4d523e8e5b6ca5b0cefa677fece","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"c680824be3d2257cbbfa3f9069712276e088e630af553b971685d6c2d068c191","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:268]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000813b80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-shoebox-hcp-cluster-v6ssms\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-shoebox-hcp-cluster-v6ssms/operationStatuses/08584113794004887021\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-pjzftjg6drs8/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-22eudp3sazqeq\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-00/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:268]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000813b80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-shoebox-hcp-cluster-v6ssms\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-shoebox-hcp-cluster-v6ssms/operationStatuses/08584113794004887021\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-pjzftjg6drs8/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-22eudp3sazqeq\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-00/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"372085e1362dc03d2432aff715c294cbccf24d97b67cc815bea5c74f1c4c7eb1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"4834d7a7ce86611f500d1d72078b83248491ffcd1d930005f197e68c04a38653","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:218]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c173c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-oidc-wi-cluster-gnz8xj\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-oidc-wi-cluster-gnz8xj/operationStatuses/08584113793892834537\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-dgmkvhdn8svv/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-s6kwp7xg43dbi\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:218]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c173c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-oidc-wi-cluster-gnz8xj\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-oidc-wi-cluster-gnz8xj/operationStatuses/08584113793892834537\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-dgmkvhdn8svv/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-s6kwp7xg43dbi\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"38fd0b890b35bbdf57dfa230d76d576863c498e1e7b695052688fdb79e1e2bf2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"2ea82ff5e0105ddc39162bc3a1fc69443f8429775e71a60d26178a993e537cf9","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:81]: failed to create customer resources for first cluster \"basic-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c52a0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster-926kkz\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-926kkz/operationStatuses/08584113797461583341\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-xz8szf6bckm4/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eqbgltuyzhpg4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-04/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:81]: failed to create customer resources for first cluster \"basic-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c52a0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster-926kkz\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-926kkz/operationStatuses/08584113797461583341\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-xz8szf6bckm4/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eqbgltuyzhpg4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-04/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"3a88ff9c2bea552ce7e69f982d9bc8a3fe40f8cc2857726ff7ab12bb802af031","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"11b0e5a286f5fd7f972ed8af20e7424d0a69e619e29ef40e658a0fe7ce3e2fc3","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:70]: failed to create customer resources for v20260901preview cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f4eaa0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-v20260901-nmdb6d\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-v20260901-nmdb6d/operationStatuses/08584113797613972324\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-s8cf494trjq4/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-xdteugpxef23k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-08/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:70]: failed to create customer resources for v20260901preview cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f4eaa0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-v20260901-nmdb6d\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-v20260901-nmdb6d/operationStatuses/08584113797613972324\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-s8cf494trjq4/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-xdteugpxef23k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-08/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"3aeea79e8feb017f06dc1e85499f863b515d2f2b70ec3f57f3c84e3398edbea6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"ca6a37a8d8bd66ca90ec1156ff2a36ff215e8cc3cee04729bc0351a1eaa5db8e","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:71]: failed to create customer resources for cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004091e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-hcp-cluster-np-128-trb4z9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-hcp-cluster-np-128-trb4z9/operationStatuses/08584113797564088180\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-q2f888mn2cwt/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3ljyjjx2n7gxo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-20/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:71]: failed to create customer resources for cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004091e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-hcp-cluster-np-128-trb4z9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-hcp-cluster-np-128-trb4z9/operationStatuses/08584113797564088180\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-q2f888mn2cwt/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3ljyjjx2n7gxo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-20/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"3b03d5fe0ca7e6202ed43a4d912c9d3679f3b745a307faf16e9e53a799136ba4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"704eb4f5538f36e002f0ab88d0c2ebd5288bd5bee60926b525641b1c59219a62","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:71]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000815280\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-delete-hcp-cluster-tz2dtt\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-delete-hcp-cluster-tz2dtt/operationStatuses/08584113794261759859\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-22/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-gxqmcjbj27xb/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rcpnvqrcxdqmo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:71]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000815280\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-delete-hcp-cluster-tz2dtt\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-delete-hcp-cluster-tz2dtt/operationStatuses/08584113794261759859\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-22/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-gxqmcjbj27xb/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rcpnvqrcxdqmo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"3bb2ab064b128a9d4cb19e83e70635ed168c918c11c27ac09e9e9cef222dfeb1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"33d5d93bef7910045cddabd91ed8d0f16ffb3abd049de20101412caa4dbdbefa","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-5-0-sqzzfn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00110a600\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-5-0-sqzzfn-lmmjjl\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-5-0-sqzzfn-lmmjjl/operationStatuses/08584113795936376304\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-jgtkss8655cz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-vx4yub7b7dgwa\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-06/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-5-0-sqzzfn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00110a600\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-5-0-sqzzfn-lmmjjl\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-5-0-sqzzfn-lmmjjl/operationStatuses/08584113795936376304\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-jgtkss8655cz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-vx4yub7b7dgwa\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-06/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"3ccc0b67acbb6edccf53c628b08e54bd2ae91e3781681d9b307f4512ea252dee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"f5544d328bdba7722d736b6d0f52967b438c0c26d6553943b71ca1a3207e4a31","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-21-xpz8fh\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010081e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-21-xpz8fh-lm75lj\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-21-xpz8fh-lm75lj/operationStatuses/08584113795344225784\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-7d9tj4kd8q9d/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-yvqbgfau4idhs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-08/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-21-xpz8fh\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010081e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-21-xpz8fh-lm75lj\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-21-xpz8fh-lm75lj/operationStatuses/08584113795344225784\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-7d9tj4kd8q9d/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-yvqbgfau4idhs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-08/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"3cddeadc6b171040d63a79ee6706d055bb011b9e7dbffab83793c02690ba60e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"c07a630ddfa4bf5e5c0aa0512829f6419194f24b58766d4a80c2eb68ccd03946","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:85]: failed to create customer resources for external auth cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b9fd20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-cluster-2dfr5b\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-cluster-2dfr5b/operationStatuses/08584113794969206545\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-13/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-fgdjlrr4zv8j/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ltdphft7fgrwg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:85]: failed to create customer resources for external auth cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b9fd20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-cluster-2dfr5b\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-cluster-2dfr5b/operationStatuses/08584113794969206545\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-13/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-fgdjlrr4zv8j/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ltdphft7fgrwg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"4872dcfea5f5c662b29b181e02b5d4b0f6646de1be7bbc847280afd240c64419","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"032d1d7a3e0d95f0228ee1c11b1d6af914ebb5716a5fa59eb161c8b7fed27753","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:283]: failed to create cluster customer resources for node-limit test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e7dae0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-node-limit-cluster-4lkc6c\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-4lkc6c/operationStatuses/08584113794966440147\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-48zl586s8mmx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:283]: failed to create cluster customer resources for node-limit test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e7dae0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-node-limit-cluster-4lkc6c\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-4lkc6c/operationStatuses/08584113794966440147\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-48zl586s8mmx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"49fc53b6c084b53fbb17d20e60d54d5b7097af4cbca6a46eeb9c01824f145980","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"fb5ce0354606977e7e3791538a4b663a81b113db149cb566343e57faaf207eb8","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:82]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c1c8c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-np-labels-taints-xc8skc\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-np-labels-taints-xc8skc/operationStatuses/08584113794310947201\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-15/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-vnr92zqdsct4/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-r6cuakilyx67k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:82]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c1c8c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-np-labels-taints-xc8skc\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-np-labels-taints-xc8skc/operationStatuses/08584113794310947201\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-15/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-vnr92zqdsct4/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-r6cuakilyx67k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"4dae835bca9717bf9547ee1d83789bf6a35b2c92692c03ebabc064726787d6d6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"54f0aecb47d52922007a03672ff611741b8fc5034419c1f438b22e2214da6935","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-21-snntjr\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000905e20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-21-snntjr-dl994l\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-21-snntjr-dl994l/operationStatuses/08584113795982950394\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-05/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-rdlcqg2zmgtz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wdrbbs4qtkscu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-21-snntjr\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000905e20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-21-snntjr-dl994l\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-21-snntjr-dl994l/operationStatuses/08584113795982950394\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-05/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-rdlcqg2zmgtz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wdrbbs4qtkscu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"4fa6ad190008883ae27b36c13bdf930590d6ebe74c16a733409b88f0f39711a8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"f929f4bbedfac90e88e7ca84f91f3733417298186cf3e83f4d4be17bfe2e51d4","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-20-4cn2wf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016ee1e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-20-4cn2wf-k4pbgs\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-4cn2wf-k4pbgs/operationStatuses/08584113795723923153\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-qt7pwt9lj829/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-mazf27myroa4e\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-00/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-20-4cn2wf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016ee1e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-20-4cn2wf-k4pbgs\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-4cn2wf-k4pbgs/operationStatuses/08584113795723923153\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-qt7pwt9lj829/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-mazf27myroa4e\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-00/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"5351fa7e7fc826454a837c609c26d8aea4bd436b0a66ba115d08ae48653e61c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"9b1e9bfdfcae58a6d83f60788f5cda0dd05af07f85b2fd0f136917dfeee0ab3c","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-5-0-p2llll\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a69d80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-5-0-p2llll-fwvg46\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-5-0-p2llll-fwvg46/operationStatuses/08584113795512069481\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-cx8fptjfl78k/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6tmfgayffrkxs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-01/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-5-0-p2llll\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a69d80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-5-0-p2llll-fwvg46\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-5-0-p2llll-fwvg46/operationStatuses/08584113795512069481\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-cx8fptjfl78k/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6tmfgayffrkxs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-01/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"565beca1363193f15bed6a758b29d858884c002be96172d75a520434780b4fdb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"c5ed5a71619d53dc86b567736741d4b55c60b49cc02e82c98980eff5d0786438","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:121]: failed to deploy managed identities for back-level version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fcfae0\u003e: \n failed waiting for deployment \"mi-cluster-ver-4-20-vzkjqw\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-ver-4-20-vzkjqw/operationStatuses/08584113795989168208\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-vlcf7czw8h2d/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-15/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:121]: failed to deploy managed identities for back-level version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fcfae0\u003e: \n failed waiting for deployment \"mi-cluster-ver-4-20-vzkjqw\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-ver-4-20-vzkjqw/operationStatuses/08584113795989168208\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-vlcf7czw8h2d/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-15/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"5bbfab560aaa6ff4e56ec2e53a0c39a8f3be45588ee649cfa23d46a71844eba3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"fd8dce044fe3c3ec2ed6837b879a1f543587662baefa69190a60f6b8acd23377","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:76]: failed to create customer resources with private key vault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010ad300\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-private-kv-cluster-r9bkv2\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-private-kv-cluster-r9bkv2/operationStatuses/08584113796499725982\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-9qdz5jkjz5x9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-had2isw6vkgre\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-22/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:76]: failed to create customer resources with private key vault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010ad300\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-private-kv-cluster-r9bkv2\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-private-kv-cluster-r9bkv2/operationStatuses/08584113796499725982\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-9qdz5jkjz5x9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-had2isw6vkgre\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-22/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"5c2513cf3ab12c825acb905760e932fad713f429714e50da1ce8d1ca1f4b038d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"ee95ed63c942a4fe05405208f698b779fd419a59453da921efa0d030f65c38da","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:101]: failed to deploy dedicated managed identities\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8d460\u003e: \n failed waiting for deployment \"mi-missing-mi-hcp-cluster-962vtg\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-missing-mi-hcp-cluster-962vtg/operationStatuses/08584113797566940213\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-rdxdv7ssp57g/providers/Microsoft.Resources/deployments/clusterMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-rdxdv7ssp57g/providers/Microsoft.Resources/deployments/clusterNonMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:101]: failed to deploy dedicated managed identities\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8d460\u003e: \n failed waiting for deployment \"mi-missing-mi-hcp-cluster-962vtg\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-missing-mi-hcp-cluster-962vtg/operationStatuses/08584113797566940213\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-rdxdv7ssp57g/providers/Microsoft.Resources/deployments/clusterMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-rdxdv7ssp57g/providers/Microsoft.Resources/deployments/clusterNonMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"5f26944453a9bcb3ee3a2584680894769b88a4305fd6fab445ec5cf4be9db2bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"cb62db27ca09cf506870028f97396da42cf6ed7737e04d83b76d35e31b696eee","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-23-zbh5mm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00096f880\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-23-zbh5mm-jkzp59\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-23-zbh5mm-jkzp59/operationStatuses/08584113795257843740\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-11/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-hcnjhtd8r27b/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-cgm32ouf327fy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-23-zbh5mm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00096f880\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-23-zbh5mm-jkzp59\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-23-zbh5mm-jkzp59/operationStatuses/08584113795257843740\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-11/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-hcnjhtd8r27b/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-cgm32ouf327fy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"5fa10079717ae5bfa5d587ae984196d8ae746a001e8329b104c5b49bd90488f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"469958b60afe99ebd686b471f988b1e0ca0f744e31bbcdce7e52c5bf5c7f1549","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-5-0-qwwn9c\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a00a80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-5-0-qwwn9c-wlvz2l\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-5-0-qwwn9c-wlvz2l/operationStatuses/08584113795128076410\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-9mtgn4gf7nhc/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-4byxptqgl7j5c\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-12/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-5-0-qwwn9c\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a00a80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-5-0-qwwn9c-wlvz2l\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-5-0-qwwn9c-wlvz2l/operationStatuses/08584113795128076410\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-9mtgn4gf7nhc/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-4byxptqgl7j5c\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-12/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"61a0f647a663bbe69873729cdd95a96fd542a56df7e8764c1563ac473ec26d41","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"538131ec214de2b12316575147e1e84a6f41ede3f84a3e93afdd963b3cfcafcc","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:87]: failed to create customer resources for TLS endpoint cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00016fa20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-tls-endpoint-hcp-cluster-z2sc66\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-tls-endpoint-hcp-cluster-z2sc66/operationStatuses/08584113797558647934\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-5ncqcmwnj49p/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oo7pyvaeowk44\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-15/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:87]: failed to create customer resources for TLS endpoint cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00016fa20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-tls-endpoint-hcp-cluster-z2sc66\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-tls-endpoint-hcp-cluster-z2sc66/operationStatuses/08584113797558647934\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-5ncqcmwnj49p/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oo7pyvaeowk44\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-15/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"620fbdb6c6a5836755ca0d43071cb141125844a8cb949b49a238bfde1aacacab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"66b6a8c8ed850f996ab5708c37e3c8119f622e5f7661cee41c168026b7225a35","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-21-dnd4px\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007c8a80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-21-dnd4px-lz842d\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-dnd4px-lz842d/operationStatuses/08584113795763065278\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-dgrk4rrfxmqr/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-h3fh53om2ygkc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-21-dnd4px\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007c8a80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-21-dnd4px-lz842d\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-dnd4px-lz842d/operationStatuses/08584113795763065278\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-dgrk4rrfxmqr/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-h3fh53om2ygkc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"684a1428b6d26dfc65dd2fc4580145593437a56936c003e7a5a596f10d67d67a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"759c7aff752bc2da8c0bbcbb05976cff2e0320d40d2167426639c5b03ea648f1","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:77]: failed to create customer resources for private KAS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef3100\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-private-kas-4r9xdd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-private-kas-4r9xdd/operationStatuses/08584113797510310178\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-14/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-jswxb8xbdgc4/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-jxdbnndsp5js2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:77]: failed to create customer resources for private KAS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef3100\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-private-kas-4r9xdd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-private-kas-4r9xdd/operationStatuses/08584113797510310178\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-14/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-jswxb8xbdgc4/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-jxdbnndsp5js2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"6bc04286df36aba088291ed430ea4756940e156d5e3fcdf4a6ab685e85b622eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"70890c7ae52a1db455deec8b3eb7490070c782e63310b6e7b685353def12711a","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:78]: failed to create customer resources for cilium CNI cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010bc880\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cilium-cl-2g89zx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cilium-cl-2g89zx/operationStatuses/08584113797437575156\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-19/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-rt2rksv85s7p/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-dve5f7wncegny\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:78]: failed to create customer resources for cilium CNI cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010bc880\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cilium-cl-2g89zx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cilium-cl-2g89zx/operationStatuses/08584113797437575156\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-19/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-rt2rksv85s7p/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-dve5f7wncegny\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"6ecf427b49e1ad9000f828fc0447765b2f8635dde6ab557c1b399512f38bb2d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"c83af701810277995349dcb52023292d5025daa4c73897a842f01114b802c577","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c29300\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-kw96zc-5l724p\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-kw96zc-5l724p/operationStatuses/08584113794227856108\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-kw96zc-lstdjv66nmvg/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w2zbgzdrtwari\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-18/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c29300\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-kw96zc-5l724p\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-kw96zc-5l724p/operationStatuses/08584113794227856108\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-kw96zc-lstdjv66nmvg/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-w2zbgzdrtwari\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-18/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"70315b277bf91e04c2f6ea8de7942f11851a41f70d824f43488df6de3f410784","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"586af51ee2e7cbcdde4edd35a4a1a29ef19ee26309fb4a112ed256320c18c4de","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:130]: failed to create customer resources for aggregated feature cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00067b320\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-agg-cluster-6fqb69\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-agg-cluster-6fqb69/operationStatuses/08584113796356952331\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-09/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-gg2tz5wqq5b9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-zkzqbkxbrvnhu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:130]: failed to create customer resources for aggregated feature cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00067b320\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-agg-cluster-6fqb69\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-agg-cluster-6fqb69/operationStatuses/08584113796356952331\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-09/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-gg2tz5wqq5b9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-zkzqbkxbrvnhu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"71065f2f234f8f717b576f1ac724fc63e62381c862f74c2bbaf1b04206fa4f90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"ce3b2648ff9b027b217801bdf840547004f2d0850e7343af604b6b1c4b1c31d0","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:88]: failed to create customer resources for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001258420\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cidr-connectivity-test-f97wpv\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-f97wpv/operationStatuses/08584113797415943541\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-13/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-tlmdcr2hfq9p/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:88]: failed to create customer resources for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001258420\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cidr-connectivity-test-f97wpv\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-f97wpv/operationStatuses/08584113797415943541\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-13/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-tlmdcr2hfq9p/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"719075e1bcf232c0060f4221261e00dbf4d3b44ad81a73a3d0a43758e8b2c84d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"ef6b465c319e2089bf73fd85a6ea8152d3ab980148dac6a645125758ec975f33","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015ca640\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-bfd6pf-66vswn\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-bfd6pf-66vswn/operationStatuses/08584113794259884121\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-bfd6pf-dhltqt8zw7sr/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-dx3utudtged7u\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-07/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015ca640\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-bfd6pf-66vswn\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-bfd6pf-66vswn/operationStatuses/08584113794259884121\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-bfd6pf-dhltqt8zw7sr/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-dx3utudtged7u\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-07/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"7b8c446a9fbd360df716d47678068a68dd4d57d67b825c095de490c1b232a4ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"087c5753d272a2ff86285ff6f45823005418b1e6f89d4b2bb703f0d6deb2094c","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000f02720\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-v5ql2zlqgx5w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000f02720\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-v5ql2zlqgx5w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"866acd83698d4b3e4a138c3d03f1a6597e563205e02faf89560a47845577b823","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"7d08c2187cf3f7a85081d3d94aa55ae175cd4f81026e76db422229194afedcc3","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:75]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000716a80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ephemeral-disk-vsscw5\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ephemeral-disk-vsscw5/operationStatuses/08584113794273516182\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-6hllwhwnc2ls/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6xts4ycasve7q\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-09/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:75]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000716a80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ephemeral-disk-vsscw5\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ephemeral-disk-vsscw5/operationStatuses/08584113794273516182\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-6hllwhwnc2ls/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6xts4ycasve7q\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-09/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"8b3c57c9a2768a5869b1ef92b9f8a9533c62d8e765542d5a9c3bacc072c3b1da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"1264aa04996713c2ad0664fc180fefec085befb80e6a3d7d8c615745c0cafacb","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:431]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014fe740\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-skip-minor-9fvzll-xr988b\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-skip-minor-9fvzll-xr988b/operationStatuses/08584113794038808904\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-9fvzll-2l748x67s24f/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wlqpvuhemxk5s\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-05/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:431]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014fe740\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-skip-minor-9fvzll-xr988b\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-skip-minor-9fvzll-xr988b/operationStatuses/08584113794038808904\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-9fvzll-2l748x67s24f/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wlqpvuhemxk5s\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-05/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"8bf8633a888c73ba63eb94c311d1e715266e4ff071d44c95cd304fa21734cb4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"5c5c223fc846287b6e5184b8ab2b2f4e1d090891bd7357cecbc12498cfaf9579","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:710]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00140c6c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-dg-minor-vnvrkq-knrpwn\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-dg-minor-vnvrkq-knrpwn/operationStatuses/08584113794346095602\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-23/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-vnvrkq-skf4kw8tskx9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-zyxd247esju2u\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:710]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00140c6c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-dg-minor-vnvrkq-knrpwn\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-dg-minor-vnvrkq-knrpwn/operationStatuses/08584113794346095602\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-23/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-vnvrkq-skf4kw8tskx9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-zyxd247esju2u\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"8ef8d64e33a3b36308cef280bdc43f0710a1ec17cbd3de522fb2147c3509c697","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"80d153140ed2a6cc68ab892fa4a4c45411338bbd76b2c74ca7b9855bd7a64595","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-23-5b4jwd\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c15860\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-23-5b4jwd-nvrdbw\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-23-5b4jwd-nvrdbw/operationStatuses/08584113795807931830\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4jlqndxm4h9r/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-caw3ygfnjzhps\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-07/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-23-5b4jwd\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c15860\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-23-5b4jwd-nvrdbw\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-23-5b4jwd-nvrdbw/operationStatuses/08584113795807931830\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4jlqndxm4h9r/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-caw3ygfnjzhps\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-07/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"8f309b37eda041dee002a2e785b9d42b0390d2db10a1c49bba94a3716b077e90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"8a7dd97750d4afe253dcc7f912d067e5dc02351e57c7ebdd90777aedf3f19660","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:79]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a0c900\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-disabled-image-registry-hcp-cluster-gbttsk\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-disabled-image-registry-hcp-cluster-gbttsk/operationStatuses/08584113795153509177\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-mz4fw89j2cdt/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qmh74zogaodhg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-16/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:79]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a0c900\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-disabled-image-registry-hcp-cluster-gbttsk\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-disabled-image-registry-hcp-cluster-gbttsk/operationStatuses/08584113795153509177\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-mz4fw89j2cdt/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qmh74zogaodhg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-16/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"9c395b4e887493bae65348a9ddae2eb5abb26740c4f7ebdcb2cad7673e8fe1ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"a6bcd28d73459594e929894c498d1e701b6c725c8fc5e8572366838164afdef9","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:104]: failed to create customer resources for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001266860\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-pullsecret-hcp-cluster-9bsxxt\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-pullsecret-hcp-cluster-9bsxxt/operationStatuses/08584113797561738888\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-zwnpp7n4fhs9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-p2h32vyw2l2ma\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-23/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:104]: failed to create customer resources for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001266860\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-pullsecret-hcp-cluster-9bsxxt\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-pullsecret-hcp-cluster-9bsxxt/operationStatuses/08584113797561738888\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-zwnpp7n4fhs9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-p2h32vyw2l2ma\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-23/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"a0b8321e72274facf30d9084f8db5965bc5d554ca8074cd3cce25533118a9e8d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"00a20048fc932b0229905148fcba92613e4cec7d16f2f63fce5ced5f3389d702","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:73]: failed to create customer resources for v20261001preview cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00106f760\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-v20261001-x6gm79\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-v20261001-x6gm79/operationStatuses/08584113797462734569\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-w8qdl2mz4kg9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-7e767rdphst3w\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-12/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:73]: failed to create customer resources for v20261001preview cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00106f760\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-v20261001-x6gm79\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-v20261001-x6gm79/operationStatuses/08584113797462734569\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-w8qdl2mz4kg9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-7e767rdphst3w\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-12/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"a5b74d75de9d85bcc115013c1df0b3bbe8305f3ff04b0a76ece4e0ca94629750","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"fa62e44b14b31ae7655734e567ec10b60e7231167b3936203b6ca0bfe48dd179","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0014ffda0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0014ffda0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ad2627fc60641392557346ff6ca5486134ab0e762b84d303f027b6f068842111","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"029ca1a09e038a2fa4b3875ee6b1d8faaee67b0d1c74d5a1f35e3c471583ada0","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:114]: failed to create customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b13100\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ingress-cert-le-5tmv7b\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ingress-cert-le-5tmv7b/operationStatuses/08584113797516953541\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-qzpjtfdnhg6c/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-nhimtfs3fueps\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:114]: failed to create customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b13100\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ingress-cert-le-5tmv7b\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ingress-cert-le-5tmv7b/operationStatuses/08584113797516953541\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-qzpjtfdnhg6c/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-nhimtfs3fueps\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"b24b95192fd01faa69036f2c8d008b8de8fff2c5a03565ad7efb004a7f78fff1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"1275667544a960574ea11de5b15d9cc89597bc9e741df1ceb42178fc733c8b11","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create customer resources for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001084920\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-autoscaling-hcp-cluster-jw828q\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-autoscaling-hcp-cluster-jw828q/operationStatuses/08584113797516221654\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-16/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-sk5txhzw7bwk/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-e4ixwyxsfjnje\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create customer resources for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001084920\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-autoscaling-hcp-cluster-jw828q\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-autoscaling-hcp-cluster-jw828q/operationStatuses/08584113797516221654\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-16/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-sk5txhzw7bwk/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-e4ixwyxsfjnje\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"b33d5e151071426c4c5cd6be3e8b801cefaf24e239b3415bca07b739c2204373","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"128bfc4ebf3e4f4ac7c72ef37dce1d6f1d2972dc28ac3084ac0eea3f1a935364","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:90]: failed to create customer resources for IDMS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000da13e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-idms-e2e-hcp-cluster-x7p8lc\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-idms-e2e-hcp-cluster-x7p8lc/operationStatuses/08584113795005880625\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-q68kljjnmbn5/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-lv7dvzspd2e5i\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-19/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:90]: failed to create customer resources for IDMS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000da13e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-idms-e2e-hcp-cluster-x7p8lc\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-idms-e2e-hcp-cluster-x7p8lc/operationStatuses/08584113795005880625\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-q68kljjnmbn5/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-lv7dvzspd2e5i\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-19/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"b62088dd4bf89ca36a0c3603380b023aeb09faf198bf070ec241a24f60d6e851","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"b3cd8ed1378cd179b5adc6dba47cf4e17d2d259a48359faed9ab53bfebd698ab","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:83]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c5c080\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-negative-tests-cluster-r7749z\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-r7749z/operationStatuses/08584113794054293332\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-03/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-mtn6lkwlzgsq/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:83]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c5c080\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-negative-tests-cluster-r7749z\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-r7749z/operationStatuses/08584113794054293332\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-03/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-mtn6lkwlzgsq/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"bdf544965754455e004b7019ad12211c2f03100691cc2b20ae4eb8ce38a3e8b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"edbb12cee0cdc493ca116717026c262450dbb9303cb1deb1d5bae75c5bd6380f","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:151]: failed to create customer resources for patch-tags cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000914b80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-patch-tags-cluster-pw89nk\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-tags-cluster-pw89nk/operationStatuses/08584113795987974979\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bkph9dv7sn7p/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fklo75htpxzxc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-21/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:151]: failed to create customer resources for patch-tags cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000914b80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-patch-tags-cluster-pw89nk\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-tags-cluster-pw89nk/operationStatuses/08584113795987974979\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-bkph9dv7sn7p/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fklo75htpxzxc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-21/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c0a46b404d7259a35a3ee205cab37da48db24c96c23256c926bf618f0e0fa2ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"6c9e3e00c4943e8c6ae844a720ee3bd59a661f127a0c9f3e1eb2ce1bbec51ae6","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:72]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e9bce0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-fips-enabled-cluster-zpl7rt\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-fips-enabled-cluster-zpl7rt/operationStatuses/08584113797321072088\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-10/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-gn2b8gjr9kwt/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3h7v3eeeqgjeu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:72]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e9bce0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-fips-enabled-cluster-zpl7rt\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-fips-enabled-cluster-zpl7rt/operationStatuses/08584113797321072088\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-10/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-gn2b8gjr9kwt/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3h7v3eeeqgjeu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c4e2e5cbd0d3bc433787675da0283902869608a3d5912cb59027b8c3d30a4792","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"74310ad621787aad91f122d4ddf1a6955a3bd453e42f4f9419b9b40e9ac8483b","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:80]: failed to create cluster customer resources for cordon test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014307c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cordon-hcp-cluster-wvnprt\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cordon-hcp-cluster-wvnprt/operationStatuses/08584113795113577924\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-17/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-5xnxcpchnklr/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3wbuyr3ac6kzo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:80]: failed to create cluster customer resources for cordon test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014307c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cordon-hcp-cluster-wvnprt\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cordon-hcp-cluster-wvnprt/operationStatuses/08584113795113577924\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-17/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-5xnxcpchnklr/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3wbuyr3ac6kzo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"ce6079e787ea9689bc17503516c3de50579982282ee366d41ef382d6b882cb72","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"121a5f9e10fa93bb1cb0ceb5ae519962392d2b7e9720f3e5ec5a5c3456f46de3","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c2cca0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-cfzv76-5v6stt\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-cfzv76-5v6stt/operationStatuses/08584113794362466667\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-cfzv76-v5jm55d72ljj/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qohedkf2u7zja\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-06/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c2cca0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-cfzv76-5v6stt\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-cfzv76-5v6stt/operationStatuses/08584113794362466667\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-cfzv76-v5jm55d72ljj/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qohedkf2u7zja\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-06/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"cfd70da43342f05995ca8887b64aa72ccf3535f80955e402a642c5d2ade6ca60","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"3202f84c16b137e078b9c7ecb07b12a42284442eaadf1a57d9b80732965d76d6","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:570]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ffc1c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-downgrade-jsfxcg-g9msqd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-downgrade-jsfxcg-g9msqd/operationStatuses/08584113794199053326\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-jsfxcg-wcdlrkd9bvdh/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ibtiavrd7ygfu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-21/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:570]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ffc1c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-downgrade-jsfxcg-g9msqd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-downgrade-jsfxcg-g9msqd/operationStatuses/08584113794199053326\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-jsfxcg-wcdlrkd9bvdh/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ibtiavrd7ygfu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-21/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"d218b732918ddbf2bf576170642e6360f58c12011190d0841e0113e178df8c24","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"c9904e37a7c79299aecd2c19efee7467df44886a34a6426023ae1408ffae1aa7","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:291]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c6ee20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-noedge-wsdcjf-kmz2kq\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-noedge-wsdcjf-kmz2kq/operationStatuses/08584113794289141624\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-wsdcjf-vd6xl6klg8wv/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-hpoi33muixkmu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-04/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:291]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c6ee20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-noedge-wsdcjf-kmz2kq\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-noedge-wsdcjf-kmz2kq/operationStatuses/08584113794289141624\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-wsdcjf-vd6xl6klg8wv/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-hpoi33muixkmu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-04/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"d29f8b74e49a49a23ffe59c34472131739cfdb6a660f132018b686b834daf1cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"9b2aa2b4206e53932073fbfb64d37b9c7758a12a200a2900f869b67ac71a624d","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-22-xvv6x2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014485c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-22-xvv6x2-9hs6th\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-22-xvv6x2-9hs6th/operationStatuses/08584113795530207961\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-10/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-nzqtchg88pjc/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-y4oqvpje3fy3s\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-22-xvv6x2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014485c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-22-xvv6x2-9hs6th\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-22-xvv6x2-9hs6th/operationStatuses/08584113795530207961\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-10/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-nzqtchg88pjc/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-y4oqvpje3fy3s\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"d8bd63f866cf7b8a480fda822ce676726e79a581611fe48c2de71410bea2dffd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"61ee045800b644443347ae3b2331e9a65e1d479e962d730dcbc98c53d281f3f5","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:74]: failed to create customer resources for patch-name cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c0a7c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-patch-name-cluster-9sqc9d\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-name-cluster-9sqc9d/operationStatuses/08584113797646453524\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-22zzphmvswcv/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rr7tdjqoygobu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-21/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:74]: failed to create customer resources for patch-name cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c0a7c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-patch-name-cluster-9sqc9d\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-name-cluster-9sqc9d/operationStatuses/08584113797646453524\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-22zzphmvswcv/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rr7tdjqoygobu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-21/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"e244007153d6b36966b8727555084e34f0c650879e853b2151331c703a33eda3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"d95ee625f921273f4468efed93cdbe7bf7598dbc0e5fd1356a3bf91649094680","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-20-2wmzq5\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00117e4c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-20-2wmzq5-p9lzt6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-20-2wmzq5-p9lzt6/operationStatuses/08584113795956074537\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-23/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-f5wgwlh8c4vw/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ldmq4dx4vhh32\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-20-2wmzq5\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00117e4c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-20-2wmzq5-p9lzt6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-20-2wmzq5-p9lzt6/operationStatuses/08584113795956074537\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-23/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-f5wgwlh8c4vw/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ldmq4dx4vhh32\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"e8dba6bbacf19c59c76f1f8b13048a4a41d680150e7f6ba383a498b485637a22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"18b554797c4bf7fcf142d9ba43f2d0e3c7d0231faa68c4e122ac2697af20af1e","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:77]: failed to create customer resources for private ingress cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd84e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-private-ingress-kpdmwx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-private-ingress-kpdmwx/operationStatuses/08584113797618485125\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-03/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-dfpk9s64d99s/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6rnsb3kbhjy5m\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:77]: failed to create customer resources for private ingress cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd84e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-private-ingress-kpdmwx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-private-ingress-kpdmwx/operationStatuses/08584113797618485125\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-03/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-dfpk9s64d99s/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6rnsb3kbhjy5m\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"ea43d9e60e42e652001309e8530cefaa1cada9125ac72348ceca36f50d004dcf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"88bf0468667696d126054af4a76866d6e508bf841aaa491e564e9b9dc090b9d1","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:78]: failed to create customer resources for ARM64 nodepool cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00108d040\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-arm64-vm-hcp-cluster-f6fqrw\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-arm64-vm-hcp-cluster-f6fqrw/operationStatuses/08584113797563017541\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-xxmw2kmjpth9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6htyoibevj7ye\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:78]: failed to create customer resources for ARM64 nodepool cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00108d040\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-arm64-vm-hcp-cluster-f6fqrw\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-arm64-vm-hcp-cluster-f6fqrw/operationStatuses/08584113797563017541\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-xxmw2kmjpth9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6htyoibevj7ye\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"ecbc7d4cd3fdacf30690fc509d6c8a7909ac00ba4c6f8453a4971651d1b8d408","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"b6d7774589a62ddba3343d513d47008b24ca7189da168c615a3350b70c6cea72","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-22-9lrcvj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017c5180\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-22-9lrcvj-ltmpzx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-22-9lrcvj-ltmpzx/operationStatuses/08584113795838273230\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-8h74rmx9b4nd/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qbe63cdkmzx74\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-04/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-22-9lrcvj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017c5180\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-22-9lrcvj-ltmpzx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-22-9lrcvj-ltmpzx/operationStatuses/08584113795838273230\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-8h74rmx9b4nd/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qbe63cdkmzx74\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-04/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"f057a729225f0bd24d39676d267b912579c07dac9af435758b199a663b5f024e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"f47fe5731873797ce03d1d26c02b76cb7c541a6c274c295b42b92e5642fd89a1","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:84]: failed to create customer resources with private key vault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d9a000\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cilium-cluster-rsh86k\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cilium-cluster-rsh86k/operationStatuses/08584113796557935367\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-stmhvp49mf2t/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-4vnk4izirdfzu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-17/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:84]: failed to create customer resources with private key vault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d9a000\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cilium-cluster-rsh86k\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cilium-cluster-rsh86k/operationStatuses/08584113796557935367\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-stmhvp49mf2t/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-4vnk4izirdfzu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-17/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"f19f8b7bc0dde800be24e4a41bcec268bd0d5344e2d89607e223e94984845d30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"0ef524766d9390776ad48ecbf26703ecb351c6ee97b9799357d914d374e4f02f","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:97]: failed to create customer resources for basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd89e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-cluster-dthblf\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-cluster-dthblf/operationStatuses/08584113797418741604\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-01/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-5bb59rzmvcrm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j3gkygvtvkvjs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:97]: failed to create customer resources for basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd89e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-cluster-dthblf\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-cluster-dthblf/operationStatuses/08584113797418741604\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-01/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-5bb59rzmvcrm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j3gkygvtvkvjs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"f5a7a262a5bbec38d56cd5ca517e0a47fe02714d4a9f53806e320052e9180a22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"beadaf19cf2680a9bc17ebe12a30ff4c469faa490d5bcc9d5ea6546c43f5c8a1","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:78]: failed to create customer resources for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000571500\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-listfdmjhw-kqcrb8\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-listfdmjhw-kqcrb8/operationStatuses/08584113795169768721\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-14/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-65zvssw2sbb2/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-igsi2gfoyf6yg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:78]: failed to create customer resources for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000571500\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-listfdmjhw-kqcrb8\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-listfdmjhw-kqcrb8/operationStatuses/08584113795169768721\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-14/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-65zvssw2sbb2/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-igsi2gfoyf6yg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"fccb602b4fbd805fc6cd4245e01e3dca3f3add2c42364beba1bd5fb65e97bbd0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","failed_at":"e2e","signature_id":"08547dd081b44323301158c280d0f3434cd05b357d34cc1bffd49f97c29ef513","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:84]: failed to create customer resources for admin credential lifecycle cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b1f100\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-admin-cred-lifecycle-958k92-469hvc\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-admin-cred-lifecycle-958k92-469hvc/operationStatuses/08584113797514814206\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-07/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-tbndf5jrz5gk/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-o7e4n5kmatesw\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:84]: failed to create customer resources for admin credential lifecycle cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b1f100\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-admin-cred-lifecycle-958k92-469hvc\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-admin-cred-lifecycle-958k92-469hvc/operationStatuses/08584113797514814206\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-07/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-tbndf5jrz5gk/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-o7e4n5kmatesw\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":59,"failure_pattern_ids":["20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54"],"matched_occurrences":59,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T04:47:20Z","started_at":"2026-09-24T04:47:20Z","completed_at":"2026-09-24T05:16:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T05:23:52.448415263Z","timing_metadata_checked_at":"2026-09-24T05:23:52.448415263Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T05:23:52.451778952Z","region_metadata_checked_at":"2026-09-24T05:23:52.451778952Z"},"failed_at":["e2e"],"failed_test_count":59,"occurrences":[{"row_id":"06580a2d612939adc4ddaff5295169cbb43c0fda3f0455dc43a978c1a2eb5028","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"925e74b7b1612e5eeea6fcbb8dbc566263b823de593165702c918d865fb5320d","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-21-vsc5jw\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00070f6a0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-21-vsc5jw-fkccmf\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-vsc5jw-fkccmf/operationStatuses/08584113804276958246\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-12/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-mzdsvtggrpzt/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-lzgahm6b5jz4u\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-21-vsc5jw\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00070f6a0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-21-vsc5jw-fkccmf\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-vsc5jw-fkccmf/operationStatuses/08584113804276958246\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-12/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-mzdsvtggrpzt/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-lzgahm6b5jz4u\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"06a5304ae91c6bd8d3c8d760593dfc4e230d70a9b5b5c465d2573134190661a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"7d875d3f62b6b594e123ead8b904ca4dd939eeb32cbb39316567bbccc9207e3a","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:85]: failed to create customer resources for external auth cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00098ec60\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-cluster-2tvq49\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-cluster-2tvq49/operationStatuses/08584113803871976925\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-xkpjgn9p2kgz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ltdphft7fgrwg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-17/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:85]: failed to create customer resources for external auth cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00098ec60\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-cluster-2tvq49\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-cluster-2tvq49/operationStatuses/08584113803871976925\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-xkpjgn9p2kgz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ltdphft7fgrwg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-17/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"082c7ed774110ed9f07d0f6bc1f7662708ce1c5072ce5fe234a39b2d5f37af3d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"12dca558bd0b511fc4a5df715c750446668edb1880da33c19093890f8b547639","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c51cc0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-rf4wqp-j4wfnb\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-rf4wqp-j4wfnb/operationStatuses/08584113803181772614\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-rf4wqp-8lml4422b28z/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eugg6mqzqllns\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-08/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c51cc0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-rf4wqp-j4wfnb\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-rf4wqp-j4wfnb/operationStatuses/08584113803181772614\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-rf4wqp-8lml4422b28z/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eugg6mqzqllns\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-08/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"09cbecab2814548d37c14097fec613b9247b9bab0045615877cc56ed5ef4b3d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"e720cadf1c011d298dc69fa164a357fd8c779a17a439688ca3a04d3526fe32d4","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:83]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c78100\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-negative-tests-cluster-hwxkxn\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-hwxkxn/operationStatuses/08584113802769766971\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-00/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-654tzd4mpbxq/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:83]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c78100\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-negative-tests-cluster-hwxkxn\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-hwxkxn/operationStatuses/08584113802769766971\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-00/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-654tzd4mpbxq/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"09e5cdea8daf2c65a9af924f0fcbf8b35ad342e905d6f46e93b684cb6aa29fc7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"7a55e2a07f0fa2912503eae7151cf0970e3f6b2d6668fab56021d109adc9d5ad","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:130]: failed to create customer resources for aggregated feature cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d1a860\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-agg-cluster-g7sdmh\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-agg-cluster-g7sdmh/operationStatuses/08584113805185771525\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-13/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-nc2jr4f2pntk/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-zkzqbkxbrvnhu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:130]: failed to create customer resources for aggregated feature cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d1a860\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-agg-cluster-g7sdmh\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-agg-cluster-g7sdmh/operationStatuses/08584113805185771525\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-13/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-nc2jr4f2pntk/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-zkzqbkxbrvnhu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"11e3f2d373a7662783bb8a0c672fcc8d87df50fef4634e40ef55ba360e3cece0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"b3fbbe913167b23f7cddbcc5beb175723e00dfbc86894399f39245e5c30ccad8","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:84]: failed to create customer resources for admin credential lifecycle cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc1bc0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-admin-cred-lifecycle-2cnwp4-7bpzvl\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-admin-cred-lifecycle-2cnwp4-7bpzvl/operationStatuses/08584113806114353804\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-lx842g284ln5/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-vjvkbm7kl4rpg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:84]: failed to create customer resources for admin credential lifecycle cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc1bc0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-admin-cred-lifecycle-2cnwp4-7bpzvl\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-admin-cred-lifecycle-2cnwp4-7bpzvl/operationStatuses/08584113806114353804\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-lx842g284ln5/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-vjvkbm7kl4rpg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"1847a6a1d398d9795c71e6e193c9fbea69d6662101333ac6d04f537fd5599593","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"b890ac9bd99e079a6b5aca1038203032bcdb627c5eafb16e539f64e5b257093a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-5-0-dhslq8\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00109a1a0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-5-0-dhslq8-5ptsrb\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-5-0-dhslq8-5ptsrb/operationStatuses/08584113804398479878\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-nj5xbnvhxvg4/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ggun3uwr6kkgc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-03/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-5-0-dhslq8\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00109a1a0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-5-0-dhslq8-5ptsrb\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-5-0-dhslq8-5ptsrb/operationStatuses/08584113804398479878\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-nj5xbnvhxvg4/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ggun3uwr6kkgc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-03/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"18bca14a91f549fde91fa1a53c7743be9d53e19b98af0ebb613fd6eb70016b13","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"1623bf2f430076fdc11544face0d86cf5da4f75f4a70fb76b1fee7e3b47f839f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-23-dknk46\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c470e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-23-dknk46-dtqz2l\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-23-dknk46-dtqz2l/operationStatuses/08584113804398489673\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-82xpjxxksvr2/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-szbmn4feb43d2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-21/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-23-dknk46\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c470e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-23-dknk46-dtqz2l\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-23-dknk46-dtqz2l/operationStatuses/08584113804398489673\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-82xpjxxksvr2/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-szbmn4feb43d2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-21/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"25f916766678db9a2c8e79079409b9e47c319a2ec1165a977cf2d85390cd038b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"3d478be5b9189d62ed2c66e9b103f81f7b86937b57c08e9d67e6ca3c30555f01","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:71]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e400c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-delete-hcp-cluster-djw8f4\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-delete-hcp-cluster-djw8f4/operationStatuses/08584113803554561663\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-6s5wqcfjnx94/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rcpnvqrcxdqmo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:71]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e400c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-delete-hcp-cluster-djw8f4\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-delete-hcp-cluster-djw8f4/operationStatuses/08584113803554561663\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-6s5wqcfjnx94/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rcpnvqrcxdqmo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"2a7a769dcb37c3bd45ce5fbf4f10e94432cbcf27e1ff4dd17e24832670d73e0c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"390eb6735fe63cb426b4f5ffec27b7b5786fa5222502d31c5b03c65ff6af65f1","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:88]: failed to create customer resources for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f50080\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cidr-connectivity-test-qqk9l9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-qqk9l9/operationStatuses/08584113806116275101\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qv6c2vf2b75m/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-14/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:88]: failed to create customer resources for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f50080\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cidr-connectivity-test-qqk9l9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-qqk9l9/operationStatuses/08584113806116275101\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qv6c2vf2b75m/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-14/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"2aefe678549f20f46098c8b0f47d520f840b41f4a042e18d8a309572ecb20c69","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"a1e87d9c7e66aaed64c7f8954d9753bc0574003ab4b1683459ff823c1d03d7c7","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create customer resources for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f14380\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-autoscaling-hcp-cluster-r9t4tf\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-autoscaling-hcp-cluster-r9t4tf/operationStatuses/08584113806116046651\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-r7frhbmt8bk7/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-e4ixwyxsfjnje\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-00/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create customer resources for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f14380\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-autoscaling-hcp-cluster-r9t4tf\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-autoscaling-hcp-cluster-r9t4tf/operationStatuses/08584113806116046651\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-r7frhbmt8bk7/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-e4ixwyxsfjnje\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-00/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"2d1ff0a2a81d83589eb793946106450576ef176774d0fad4ddbee931527b4859","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"a70700610a1b0fb554b19357597279649c0cb3ed6f5aee8c998069ad0ae49628","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:80]: failed to create customer resources for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001284cc0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cx-rg-hcp-cluster-vctv7k\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cx-rg-hcp-cluster-vctv7k/operationStatuses/08584113806113617110\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-23/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-hbn74zshqfgs/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fmxs36wu5sow2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:80]: failed to create customer resources for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001284cc0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cx-rg-hcp-cluster-vctv7k\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cx-rg-hcp-cluster-vctv7k/operationStatuses/08584113806113617110\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-23/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-hbn74zshqfgs/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fmxs36wu5sow2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"2e842882a2338d821d02f8e08fa922a4a9298d7915019ef460d4e180433ef52a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"0b5092b1753b51d12ff4c29870adbc656c13e6976d861905264ee7910638f41f","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:93]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010216e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-autoscale-cluster-szsxcs\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-autoscale-cluster-szsxcs/operationStatuses/08584113803873470718\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-skcrdsr4h57j/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ssrwqrnhyyabs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-22/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:93]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010216e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-autoscale-cluster-szsxcs\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-autoscale-cluster-szsxcs/operationStatuses/08584113803873470718\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-skcrdsr4h57j/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ssrwqrnhyyabs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-22/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"32803c45e013c91330a99453d7190497d49b32181d5a59af20d33c149edc3034","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"d6768673dc1158caaa55d77db95d18d312042e5c93f5efa86e3fda846977c9a9","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:79]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c53600\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-disabled-image-registry-hcp-cluster-knj66q\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-disabled-image-registry-hcp-cluster-knj66q/operationStatuses/08584113803896599497\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-20/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-5zxmx4mxkgnm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qmh74zogaodhg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:79]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c53600\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-disabled-image-registry-hcp-cluster-knj66q\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-disabled-image-registry-hcp-cluster-knj66q/operationStatuses/08584113803896599497\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-20/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-5zxmx4mxkgnm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qmh74zogaodhg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"43ecef9caba23aeb9b92dde1ea5724ea12760981b47d58408fcda830821c7acc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"502c008e6a6776d9a96bd10fd01cc6e2363833b568e55b82bb09d9627e16887e","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-22-dqs5k4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014a6d40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-22-dqs5k4-pqc4k8\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-22-dqs5k4-pqc4k8/operationStatuses/08584113804282173996\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-rrxd587pzlwh/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-76eghfhmal6jk\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-10/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-22-dqs5k4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014a6d40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-22-dqs5k4-pqc4k8\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-22-dqs5k4-pqc4k8/operationStatuses/08584113804282173996\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-rrxd587pzlwh/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-76eghfhmal6jk\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-10/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"45fcb8262c0e24dc058d92333b86a3968b023795c6b9ce245284c83d41387e43","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"0304d759fd7f723fc6fce46557757270276c6ef816162705b4c141fd03f33c2d","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:74]: failed to create customer resources for patch-name cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b1a5c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-patch-name-cluster-q7xjnw\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-name-cluster-q7xjnw/operationStatuses/08584113806111250765\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-10/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-cfmdxs2h5mz4/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rr7tdjqoygobu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:74]: failed to create customer resources for patch-name cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b1a5c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-patch-name-cluster-q7xjnw\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-name-cluster-q7xjnw/operationStatuses/08584113806111250765\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-10/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-cfmdxs2h5mz4/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rr7tdjqoygobu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"51eca3a2ee0ef45bc6fe7ba472a9df059c96381092f4a3c68580613bf599b326","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"23b8a93a781e18d248ec90141ae1a827a89330a7e7a6023ebb81f712b1aa55d8","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-22-5hfq9c\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001012860\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-22-5hfq9c-gktmn8\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-22-5hfq9c-gktmn8/operationStatuses/08584113804128623031\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-k5skrl4wfdrn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-b7qhh3pt3l6g4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-00/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-22-5hfq9c\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001012860\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-22-5hfq9c-gktmn8\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-22-5hfq9c-gktmn8/operationStatuses/08584113804128623031\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-k5skrl4wfdrn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-b7qhh3pt3l6g4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-00/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"5547decd17b46277777c35ba051b30a5a0ae4820b52c92773dad2dcb25eb8911","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"fc6f49312f9cd176b6a5c0c05d229dbd5b8a414551493a3fe3a7e14f32ebb667","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001000780\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-fgw5w2-9mhxrf\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-fgw5w2-9mhxrf/operationStatuses/08584113803282955814\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-fgw5w2-djf5mhdkmtxn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fgbekhquzb3iq\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-06/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001000780\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-fgw5w2-9mhxrf\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-fgw5w2-9mhxrf/operationStatuses/08584113803282955814\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-fgw5w2-djf5mhdkmtxn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fgbekhquzb3iq\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-06/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"5e97d3c9ce89afd3bb82275161656b8b10b5e108174ba7449374fef9d08e0259","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"4d8344447dfa277aaac9f86488fc2fba58be5beec2d8d53a92b1f78394fdde32","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:570]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004b8700\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-downgrade-rsjgrk-w7nqbj\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-downgrade-rsjgrk-w7nqbj/operationStatuses/08584113802912270287\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-rsjgrk-crbww5k2htkx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-tgmdzqh5zlmc2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-03/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:570]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004b8700\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-downgrade-rsjgrk-w7nqbj\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-downgrade-rsjgrk-w7nqbj/operationStatuses/08584113802912270287\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-rsjgrk-crbww5k2htkx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-tgmdzqh5zlmc2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-03/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"6b3c8fa2008921747f644cc6fc85bdde5d57bdf01be465141edd6f9b3830d517","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"725f05032bea5bd0ce516b91dba1d1346b457790f2cf4f5d22ff28e0e5fca8e2","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:82]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001690280\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-np-labels-taints-4spgtm\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-np-labels-taints-4spgtm/operationStatuses/08584113803454899552\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-13/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-jm8859lxjth9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-r6cuakilyx67k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:82]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001690280\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-np-labels-taints-4spgtm\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-np-labels-taints-4spgtm/operationStatuses/08584113803454899552\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-13/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-jm8859lxjth9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-r6cuakilyx67k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"7c4c6259a42f5b3506e0d4f0a0cf43d9f06e9150e686e328ea03f77ccfa286c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"5bd96944e5c33f0e4a58b08b961f8094f9e02ddbb7628a6b1e92dab670f4873d","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:431]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c89200\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-skip-minor-gx8gpn-ntvf2f\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-skip-minor-gx8gpn-ntvf2f/operationStatuses/08584113802994503918\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-01/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-gx8gpn-6np5x59hjxbc/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ikzua3x7s327g\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:431]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c89200\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-skip-minor-gx8gpn-ntvf2f\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-skip-minor-gx8gpn-ntvf2f/operationStatuses/08584113802994503918\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-01/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-gx8gpn-6np5x59hjxbc/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ikzua3x7s327g\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"7da50fc5c368b3766df66f80b2b2ce2859980f364df17be9554ea419c8068873","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"ea1a9664879960e3782bac1bab4881ef33ed58c68b27f3b3b3ea9ee942bc8f8e","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:77]: failed to create customer resources for private ingress cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e19840\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-private-ingress-s6brt9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-private-ingress-s6brt9/operationStatuses/08584113806112848721\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-rvz7ml6rm6pn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6rnsb3kbhjy5m\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-12/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:77]: failed to create customer resources for private ingress cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e19840\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-private-ingress-s6brt9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-private-ingress-s6brt9/operationStatuses/08584113806112848721\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-rvz7ml6rm6pn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6rnsb3kbhjy5m\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-12/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"8202e7d68008839bd137994a7adef99d8c2133e58d2060c6bd7c293dd839ad82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"d762de026942b3816437197f33b8846d8d2d9b251a4615d40901dd13cdf8719a","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:72]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce6a60\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-fips-enabled-cluster-hj2qcn\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-fips-enabled-cluster-hj2qcn/operationStatuses/08584113806117189919\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-11/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-jms9dt5jw97h/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3h7v3eeeqgjeu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:72]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce6a60\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-fips-enabled-cluster-hj2qcn\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-fips-enabled-cluster-hj2qcn/operationStatuses/08584113806117189919\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-11/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-jms9dt5jw97h/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3h7v3eeeqgjeu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"82d23d4fe8f37adc3fc530ebac87f1add617388ec82207a51c55ebd16fe98f60","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"5ef6cd0378df1b88f9a1e441ce1513797d9143870e2e548690d877816da13ed4","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:283]: failed to create cluster customer resources for node-limit test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c8fe40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-node-limit-cluster-mkw8wg\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-mkw8wg/operationStatuses/08584113803795290145\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-tknq4cvl4w4l/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-23/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:283]: failed to create cluster customer resources for node-limit test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c8fe40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-node-limit-cluster-mkw8wg\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-mkw8wg/operationStatuses/08584113803795290145\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-tknq4cvl4w4l/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-23/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"87c39cc79ab17efbc71b5f1cf1d759edd5ea4d05f175e257cfd2015286f85def","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"2d4c586a289252410c5432cc2b9b0821bd3ea82888527dba0a2f593c88ba237a","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:78]: failed to create customer resources for ARM64 nodepool cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015107a0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-arm64-vm-hcp-cluster-2d4bf7\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-arm64-vm-hcp-cluster-2d4bf7/operationStatuses/08584113806114468713\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-wrdgbrcf2mft/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6htyoibevj7ye\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-03/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:78]: failed to create customer resources for ARM64 nodepool cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015107a0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-arm64-vm-hcp-cluster-2d4bf7\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-arm64-vm-hcp-cluster-2d4bf7/operationStatuses/08584113806114468713\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-wrdgbrcf2mft/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6htyoibevj7ye\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-03/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"8aeec8463053aa40c0639bddbc75312b83be51940de329b467c26a1d3577bca0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"c17a5e36a9b108e239e9cbfe37ad374080446eb38001b2f19c480dc30ff0f28d","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:73]: failed to create customer resources for v20261001preview cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000515120\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-v20261001-tcn8tt\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-v20261001-tcn8tt/operationStatuses/08584113806116665011\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-nkwfqtlk4dqg/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-7e767rdphst3w\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-01/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:73]: failed to create customer resources for v20261001preview cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000515120\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-v20261001-tcn8tt\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-v20261001-tcn8tt/operationStatuses/08584113806116665011\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-nkwfqtlk4dqg/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-7e767rdphst3w\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-01/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"93b4522534f22f2e91cabdafd20a99861e48b9a638b6e55436655ff102fdec4a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"635d55a9178bb2083d65e2a4ad10093076f6eec948e15b30f0eb5b0f72168cb5","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008a27a0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-8l7wc7-twjjnz\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-8l7wc7-twjjnz/operationStatuses/08584113803024332837\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-8l7wc7-jq8k2zdmzm65/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-cltijclxbxmue\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008a27a0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-8l7wc7-twjjnz\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-8l7wc7-twjjnz/operationStatuses/08584113803024332837\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-8l7wc7-jq8k2zdmzm65/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-cltijclxbxmue\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"9dd297cfcde0b705a1e3a05a36fceac93da2ef80a8635e59fefa96e4d1251224","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"890e1a56439cee473baa172f8e4a9c46c0855344ecca13d7f575a0200da7b939","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:76]: failed to create customer resources with private key vault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011bc1e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-private-kv-cluster-rbg7tm\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-private-kv-cluster-rbg7tm/operationStatuses/08584113805186906568\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-t8475hwxlvk4/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-had2isw6vkgre\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:76]: failed to create customer resources with private key vault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011bc1e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-private-kv-cluster-rbg7tm\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-private-kv-cluster-rbg7tm/operationStatuses/08584113805186906568\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-t8475hwxlvk4/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-had2isw6vkgre\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"a48c22e59cfa81af57ad95c9e8705eb27bc06a8c5af250c0e4c5be6aba43a613","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"07068c99aba746939ca0e7fd85f8d0268d89c19b415bcc1ee9c1c9a86fbe9b05","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:90]: failed to create customer resources for IDMS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e57d80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-idms-e2e-hcp-cluster-nc2q4j\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-idms-e2e-hcp-cluster-nc2q4j/operationStatuses/08584113803895991084\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-19/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-mdbl6wxhbzjx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-lv7dvzspd2e5i\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:90]: failed to create customer resources for IDMS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e57d80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-idms-e2e-hcp-cluster-nc2q4j\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-idms-e2e-hcp-cluster-nc2q4j/operationStatuses/08584113803895991084\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-19/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-mdbl6wxhbzjx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-lv7dvzspd2e5i\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"a4ed8c9335a8673ad17f0ceb049f6de554cb764a42bdfb0900f0c551067b99ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"63d34142a504e05f9036dd1c69be15d31757de56dafb77a0d0f0177274375c52","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:104]: failed to create customer resources for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b32720\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-pullsecret-hcp-cluster-7w52q9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-pullsecret-hcp-cluster-7w52q9/operationStatuses/08584113806109025158\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-v22dzqxkj6qc/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-p2h32vyw2l2ma\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-07/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:104]: failed to create customer resources for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b32720\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-pullsecret-hcp-cluster-7w52q9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-pullsecret-hcp-cluster-7w52q9/operationStatuses/08584113806109025158\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-v22dzqxkj6qc/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-p2h32vyw2l2ma\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-07/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"aa08a1415f4c0bc0604843627580cd3ec34495461b123bf06ca98584c49edbeb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"a0382390c3701f1ad5bf1bd591fd3915695d3a07b9d0cc99388388748d139457","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:75]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000894a80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ephemeral-disk-f9x9n9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ephemeral-disk-f9x9n9/operationStatuses/08584113803560812998\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-qsvvp4zd6rfh/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6xts4ycasve7q\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-04/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:75]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000894a80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ephemeral-disk-f9x9n9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ephemeral-disk-f9x9n9/operationStatuses/08584113803560812998\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-qsvvp4zd6rfh/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6xts4ycasve7q\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-04/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"ad52314d6abc12d8d364fb22f44048ec178c5def70ade0e83e9a2ec328ec85bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"0bd173944720937f7045f39d5686ebbee16cf55a1935292dc7a1905cc174def2","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-5-0-ncrdgm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00032fc40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-5-0-ncrdgm-fbq9mg\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-5-0-ncrdgm-fbq9mg/operationStatuses/08584113804280814283\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-rfdmx8dkwsjv/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-4vtavx33era4k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-11/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-5-0-ncrdgm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00032fc40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-5-0-ncrdgm-fbq9mg\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-5-0-ncrdgm-fbq9mg/operationStatuses/08584113804280814283\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-rfdmx8dkwsjv/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-4vtavx33era4k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-11/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"b7f3ec7fcca69a8781d15c769c01692a272404ee4d2f781ea9407399fee35e7c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"982fb021ef82674968423b11c2470423c0daeb157256c664d1becb93e8210a75","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:78]: failed to create customer resources for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c0eae0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-list9dpjtf-jf5xkr\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-list9dpjtf-jf5xkr/operationStatuses/08584113803972375281\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-18/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-64sp6cvpwd29/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-adn3hfaas23n6\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:78]: failed to create customer resources for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c0eae0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-list9dpjtf-jf5xkr\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-list9dpjtf-jf5xkr/operationStatuses/08584113803972375281\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-18/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-64sp6cvpwd29/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-adn3hfaas23n6\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"bad8daa81c9b23a70564e61fa7efaeaadd35cc0d956ee1166c5aa00cf1d7febc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"96b896ebd4bd97e76579fe89b0bf3e2bf9f53a8625a52c8127ac1a1e8e6b3b60","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-20-xhggfs\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010561e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-20-xhggfs-2wkbx6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-20-xhggfs-2wkbx6/operationStatuses/08584113804398595348\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-4qk5llt4fz6d/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-a7jlkrfbfsqok\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-09/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-20-xhggfs\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010561e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-20-xhggfs-2wkbx6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-20-xhggfs-2wkbx6/operationStatuses/08584113804398595348\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-4qk5llt4fz6d/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-a7jlkrfbfsqok\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-09/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"bcb31f4b644ae137bdcc05adf30c4156d5b8958d589c79af6909f6fb0d0c2574","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"843855ae1f0f4f6cd0e66370447e4b9beddc6fb7e68d7e5e02492021371b9652","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:71]: failed to create customer resources for cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010f8700\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-hcp-cluster-np-128-j574p2\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-hcp-cluster-np-128-j574p2/operationStatuses/08584113806104875981\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-06/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-kglphbbzhqst/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3ljyjjx2n7gxo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:71]: failed to create customer resources for cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010f8700\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-hcp-cluster-np-128-j574p2\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-hcp-cluster-np-128-j574p2/operationStatuses/08584113806104875981\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-06/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-kglphbbzhqst/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3ljyjjx2n7gxo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"bfe6897a1c86dc763a58aaca296f428b134a8597c50d24b5c3debf2c78a70411","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"712f9dd3b637bec2b5625a0186e7395b21baa4927eda84c10d89e02f5b9396d5","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:114]: failed to create customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019897c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ingress-cert-le-s6pz6g\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ingress-cert-le-s6pz6g/operationStatuses/08584113806109906954\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-jhxdz7ppd98q/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-nhimtfs3fueps\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-20/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:114]: failed to create customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019897c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ingress-cert-le-s6pz6g\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ingress-cert-le-s6pz6g/operationStatuses/08584113806109906954\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-jhxdz7ppd98q/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-nhimtfs3fueps\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-20/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c18307caa8d2dba3614e4c06e2e325753c0ec28b124f95cc352ed017ca8c3fd4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"50fa57f55c832fa17b6f7248a05fabef16d409042f554f1463f8a8292757b10c","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:70]: failed to create customer resources for v20260901preview cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000820260\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-v20260901-dp7qzd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-v20260901-dp7qzd/operationStatuses/08584113806116691335\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-zrfd45q4c72q/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-xdteugpxef23k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-09/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:70]: failed to create customer resources for v20260901preview cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000820260\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-v20260901-dp7qzd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-v20260901-dp7qzd/operationStatuses/08584113806116691335\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-zrfd45q4c72q/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-xdteugpxef23k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-09/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c381755c055abab9813ac9b497f53feda072d0021e03f56fb985b7cd80b2bcab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"037c80fef56df03fab8fd20ad0d7c86fd678553fe25a7b65681038b32e94955c","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-23-ct2xd4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f810c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-23-ct2xd4-2pfhjt\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-23-ct2xd4-2pfhjt/operationStatuses/08584113804018242618\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-5jtfxzb57jbp/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oko5an6hosdhk\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-14/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-23-ct2xd4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f810c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-23-ct2xd4-2pfhjt\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-23-ct2xd4-2pfhjt/operationStatuses/08584113804018242618\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-5jtfxzb57jbp/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oko5an6hosdhk\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-14/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c44f98f2bc5f453f658840c00826f1386779db636ebeecfa52c6ea54c5bd81fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"cab3a7cd895f69adf001c052761d92c86ca6c64727e980c568920541d66b6693","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-20-7zmhsv\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a0c420\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-20-7zmhsv-7scgzz\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-7zmhsv-7scgzz/operationStatuses/08584113804277874316\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-tr2g922h2hqn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-4udo43sc7vttm\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-07/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-20-7zmhsv\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a0c420\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-20-7zmhsv-7scgzz\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-7zmhsv-7scgzz/operationStatuses/08584113804277874316\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-tr2g922h2hqn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-4udo43sc7vttm\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-07/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c6956a13266dde2ead4219741afa40742370bac720edd8260cadcef39190e790","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"91478fa5c60e4c1a9e847cc7a2a672128b7fd9e86bf7c5aea99592fde71e73bb","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:87]: failed to create customer resources for TLS endpoint cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006dda40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-tls-endpoint-hcp-cluster-79w9fh\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-tls-endpoint-hcp-cluster-79w9fh/operationStatuses/08584113806216845740\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-08/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-h527s8x62fj7/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oo7pyvaeowk44\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:87]: failed to create customer resources for TLS endpoint cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006dda40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-tls-endpoint-hcp-cluster-79w9fh\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-tls-endpoint-hcp-cluster-79w9fh/operationStatuses/08584113806216845740\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-08/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-h527s8x62fj7/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oo7pyvaeowk44\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c6d9dfa11883331c49f4982bfb6fd65109d9ee7eb0e61d3e9a976f3ec836ca98","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"dce118f0d6bf88fb6228496d672d2f34dcd0097ae26939c9a39426a75b29b854","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-22-fxdz9q\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001018360\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-22-fxdz9q-vhkwqs\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-22-fxdz9q-vhkwqs/operationStatuses/08584113804540699539\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-rcz2tr8hxdph/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-iqn6stss77wf6\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-22-fxdz9q\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001018360\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-22-fxdz9q-vhkwqs\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-22-fxdz9q-vhkwqs/operationStatuses/08584113804540699539\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-rcz2tr8hxdph/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-iqn6stss77wf6\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"d12d015578236c977584d40aebe54965e5c1d489634301de6371305864f54c50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"9bb8908c4059fc50132ad3029a7c2b0cbf42b2c3896b45ab04df63c56500a523","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:218]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f99d00\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-oidc-wi-cluster-2xqhrd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-oidc-wi-cluster-2xqhrd/operationStatuses/08584113802810164928\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-11/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-s7nl4n9bv5dn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-s6kwp7xg43dbi\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:218]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f99d00\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-oidc-wi-cluster-2xqhrd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-oidc-wi-cluster-2xqhrd/operationStatuses/08584113802810164928\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-11/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-s7nl4n9bv5dn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-s6kwp7xg43dbi\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"d212cbf53c16bb8b73a038432b4ede456a001fd5e232691abc8c9fc2317d0794","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"091b2e0d12e6d8ebf4284168ae4c1d8f0085c61f2ad326ef5833e98ccf402f45","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:78]: failed to create customer resources for cilium CNI cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010f48e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cilium-cl-8ps7nq\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cilium-cl-8ps7nq/operationStatuses/08584113806111815531\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-xf8c9fwbfv9z/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-dve5f7wncegny\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-21/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:78]: failed to create customer resources for cilium CNI cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010f48e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cilium-cl-8ps7nq\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cilium-cl-8ps7nq/operationStatuses/08584113806111815531\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-xf8c9fwbfv9z/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-dve5f7wncegny\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-21/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"d4926cbf34dc5475e51fa6fc2130c66ed4244af7b26cb1a7d16a0b8915a58479","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"83f41dede741122effea9bed86f31cb70cc79e3fe60190c74a81ccf095575a1e","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:84]: failed to create customer resources with private key vault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001218300\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cilium-cluster-5pfptn\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cilium-cluster-5pfptn/operationStatuses/08584113805189245935\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-04/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-z5mnjxckjl2l/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-4vnk4izirdfzu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:84]: failed to create customer resources with private key vault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001218300\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cilium-cluster-5pfptn\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cilium-cluster-5pfptn/operationStatuses/08584113805189245935\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-04/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-z5mnjxckjl2l/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-4vnk4izirdfzu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"d5064b53f75183c93f93b5a5f1101cb09a4731de68f7fb9b32f91bc1bb764de2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"5b26974e1da059929c65b663e256e8e0b0cadfd7ed25c9ebdf8345a70426d39a","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:80]: failed to create cluster customer resources for cordon test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00070df60\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cordon-hcp-cluster-tkt7nm\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cordon-hcp-cluster-tkt7nm/operationStatuses/08584113803709711286\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-lxhcqp5g8zsj/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3wbuyr3ac6kzo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-05/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:80]: failed to create cluster customer resources for cordon test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00070df60\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cordon-hcp-cluster-tkt7nm\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cordon-hcp-cluster-tkt7nm/operationStatuses/08584113803709711286\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-lxhcqp5g8zsj/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3wbuyr3ac6kzo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-05/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"d9f6f206f212333b92de06be2e5cf2276b63ceb0591b5f797810426833739214","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"4e29cfc3d96fb63ef07b9b320d9429e79b35c452ece26e453a45510107fb8132","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:710]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000697420\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-dg-minor-ng7fj6-gx6vsk\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-dg-minor-ng7fj6-gx6vsk/operationStatuses/08584113802810199751\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-ng7fj6-q2lfhm7n5bk7/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-4k3ew4a3xlt2e\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-10/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:710]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000697420\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-dg-minor-ng7fj6-gx6vsk\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-dg-minor-ng7fj6-gx6vsk/operationStatuses/08584113802810199751\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-ng7fj6-q2lfhm7n5bk7/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-4k3ew4a3xlt2e\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-10/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"da742281a034f2979e2e7442686407cd67c1c62d4dcfdf36bf8da1b19ee94f22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"6722686da3723d8b0bf2c0c56e808e2f4cf683cdb005fcd9744c5aaecd3e702c","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:121]: failed to deploy managed identities for back-level version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001109220\u003e: \n failed waiting for deployment \"mi-cluster-ver-4-20-sv2j7p\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-ver-4-20-sv2j7p/operationStatuses/08584113804722825144\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-06/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-mgn4698wxq98/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:121]: failed to deploy managed identities for back-level version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001109220\u003e: \n failed waiting for deployment \"mi-cluster-ver-4-20-sv2j7p\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-ver-4-20-sv2j7p/operationStatuses/08584113804722825144\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-06/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-mgn4698wxq98/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"e1f446da275189b26a870808a90ef7aabf1ad3c33f33d56a6a9c9e446a3ba032","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"5ca763c04ba811fe44fa262d90c103eac78fc982660138b2fee9f970c2edb86f","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:291]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e81ee0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-noedge-gn68jq-lvkpvg\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-noedge-gn68jq-lvkpvg/operationStatuses/08584113803021091151\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-gn68jq-sz2zh9b526g4/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ij42gficifcsk\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-21/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:291]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e81ee0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-noedge-gn68jq-lvkpvg\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-noedge-gn68jq-lvkpvg/operationStatuses/08584113803021091151\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-gn68jq-sz2zh9b526g4/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ij42gficifcsk\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-21/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"e29fcd1e1e86640021861d752e3fff99d9def15f88c689248f0a6dc5a0a05eb5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"4c47478a4a085b6edffed2a7105111b2464ffaf9972aae8beea63b50843450d3","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000ce62a0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000ce62a0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e9f99bc67233950943bb6b02d0b04e83da2e20ff16e9702f8b01d2c204a09d25","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"a7ea24781db537110c35917d98b4f72ecdedac8d30c9c33371781202312a2f05","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:268]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d6a780\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-shoebox-hcp-cluster-dvbk2t\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-shoebox-hcp-cluster-dvbk2t/operationStatuses/08584113802808793038\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-6jlj7g4z6pp2/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-22eudp3sazqeq\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-09/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:268]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d6a780\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-shoebox-hcp-cluster-dvbk2t\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-shoebox-hcp-cluster-dvbk2t/operationStatuses/08584113802808793038\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-6jlj7g4z6pp2/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-22eudp3sazqeq\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-09/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"e9ff973ca75f7aa1af357784e1fa8ec9b4c10888a9515d39cf5adb05426ec84c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"994e65619d43c6d0e316a7b351700514908589ec59750bc0f6395e6be91c7836","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:77]: failed to create customer resources for private KAS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004ee660\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-private-kas-s9c9qb\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-private-kas-s9c9qb/operationStatuses/08584113806116772609\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-f6vgfpj5z76c/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-jxdbnndsp5js2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-22/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:77]: failed to create customer resources for private KAS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004ee660\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-private-kas-s9c9qb\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-private-kas-s9c9qb/operationStatuses/08584113806116772609\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-f6vgfpj5z76c/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-jxdbnndsp5js2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-22/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"ed484e85c22c3f7390334f1d93e3a05542cc66f0174c92590b48557b8a652c42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"e4854e50e27a97b30d706a8853005f6b6d013f9d8ed46f6cafc31a004843dc2f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-21-pslr2c\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c8fa40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-21-pslr2c-g2769k\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-21-pslr2c-g2769k/operationStatuses/08584113804400098279\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-5lr6bhjf8l8d/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fmduqqi6yc6ls\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-01/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-21-pslr2c\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c8fa40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-21-pslr2c-g2769k\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-21-pslr2c-g2769k/operationStatuses/08584113804400098279\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-5lr6bhjf8l8d/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fmduqqi6yc6ls\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-01/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"f1cd1ae7e16cfdb5bf121b98c5cd44997941595cb50e904d35918619ce5d6071","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"e7f4fee1c466f0459af35e40872aa4b567ea48855febfe78971983ee7706ff97","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-21-69w4xz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010734c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-21-69w4xz-2th7h6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-21-69w4xz-2th7h6/operationStatuses/08584113804026603480\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-15/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-97lmjm8gbhml/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-evvqr3vrpmzr2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-21-69w4xz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010734c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-21-69w4xz-2th7h6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-21-69w4xz-2th7h6/operationStatuses/08584113804026603480\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-15/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-97lmjm8gbhml/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-evvqr3vrpmzr2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"f314621554e85253b02ca97e4e2ef316ff5af6b45aee73023322453d7a4b03a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"3c862361541ceae5c062b4ba0dc064b68cae0b5d598e706f6762f085e0607306","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-5-0-2lhggq\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bd9620\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-5-0-2lhggq-7dffjn\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-5-0-2lhggq-7dffjn/operationStatuses/08584113804022878880\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-fqhpj4gfndmh/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-zmtivx2uqqptm\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-16/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-5-0-2lhggq\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bd9620\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-5-0-2lhggq-7dffjn\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-5-0-2lhggq-7dffjn/operationStatuses/08584113804022878880\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-fqhpj4gfndmh/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-zmtivx2uqqptm\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-16/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"f42a252e6d9161e641f6a24acae008326c5fa3af344f4e27d6ee15aba27fe9e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"52b1633f73d41b43932bc93f25195e310e768881694919964ac5f253eaea7842","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000ef6450\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-8zb6m4wbcswd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000ef6450\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-8zb6m4wbcswd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f98d4dc3f6932f102b9bdd18771ad61623b9f9d0e8d738b5fd8d9c89d1ffbeb5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"0be8a0f9e257eed68f104074868b8b17f7d7098c3a69d645d20758aa0235f203","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:97]: failed to create customer resources for basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004bf3a0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-cluster-r5dnns\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-cluster-r5dnns/operationStatuses/08584113806114250372\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-8czpd2bttnxj/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j3gkygvtvkvjs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-16/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:97]: failed to create customer resources for basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004bf3a0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-cluster-r5dnns\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-cluster-r5dnns/operationStatuses/08584113806114250372\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-8czpd2bttnxj/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j3gkygvtvkvjs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-16/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"f9d591b9d4604ee23b0149df39aa34d294214bbaba7807a6b815274e9488f519","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"26253298960cf5b479585bcc88d9b58ed6d2ef483db264c7c7260d823ea27662","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:81]: failed to create customer resources for first cluster \"basic-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010fb4e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster-52qtxz\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-52qtxz/operationStatuses/08584113806115129746\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-4fxgzntt4rxq/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eqbgltuyzhpg4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-17/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:81]: failed to create customer resources for first cluster \"basic-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010fb4e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster-52qtxz\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-52qtxz/operationStatuses/08584113806115129746\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-4fxgzntt4rxq/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eqbgltuyzhpg4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-17/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"fe49662a0533dcb17e191dc88e14b4f2290df0ac2eb300e63df0bfea714f0253","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"641eb4af1997048f835df858f7f7bef584d41aee6bb04fe645ddcf1e44aa00e3","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:151]: failed to create customer resources for patch-tags cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006ef8c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-patch-tags-cluster-ntlx9n\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-tags-cluster-ntlx9n/operationStatuses/08584113804793509594\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-mntvs5bfs2vm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fklo75htpxzxc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-08/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:151]: failed to create customer resources for patch-tags cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006ef8c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-patch-tags-cluster-ntlx9n\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-tags-cluster-ntlx9n/operationStatuses/08584113804793509594\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-mntvs5bfs2vm/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fklo75htpxzxc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-08/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"feb8865636aab81baecccd199f630307387993ac26587ac3bf64a072cdc82d80","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","failed_at":"e2e","signature_id":"aaca3b66cd93923710b64ada020932cb93aacc5dd79493313bc4c64ce07c0aad","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:101]: failed to deploy dedicated managed identities\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0003577a0\u003e: \n failed waiting for deployment \"mi-missing-mi-hcp-cluster-qbdg8q\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-missing-mi-hcp-cluster-qbdg8q/operationStatuses/08584113806112641677\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-k57vjlqvw6kx/providers/Microsoft.Resources/deployments/clusterNonMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-k57vjlqvw6kx/providers/Microsoft.Resources/deployments/clusterMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:101]: failed to deploy dedicated managed identities\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0003577a0\u003e: \n failed waiting for deployment \"mi-missing-mi-hcp-cluster-qbdg8q\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-missing-mi-hcp-cluster-qbdg8q/operationStatuses/08584113806112641677\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-k57vjlqvw6kx/providers/Microsoft.Resources/deployments/clusterNonMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-k57vjlqvw6kx/providers/Microsoft.Resources/deployments/clusterMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":59,"failure_pattern_ids":["20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54"],"matched_occurrences":59,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T02:00:13Z","started_at":"2026-09-24T02:00:13Z","completed_at":"2026-09-24T02:10:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T02:15:13.91325451Z","timing_metadata_checked_at":"2026-09-24T02:15:13.91325451Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T02:15:13.920382187Z","region_metadata_checked_at":"2026-09-24T02:15:13.920382187Z"},"failed_at":["e2e"],"failed_test_count":59,"occurrences":[{"row_id":"0171e488a4a2fc25fae6db9d27ab6d2255066b26fc3de783c76606f86220bf1c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"24a84c9669e3bf30f2efd1876022065618eeea5fd4fc89076c6ac044a71d108a","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c98200\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-x48fn7-f4mhn6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-x48fn7-f4mhn6/operationStatuses/08584113912430214940\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-11/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-x48fn7-rx4kn7h7l62p/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-igmhstzhz2tea\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c98200\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-x48fn7-f4mhn6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-x48fn7-f4mhn6/operationStatuses/08584113912430214940\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-11/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-x48fn7-rx4kn7h7l62p/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-igmhstzhz2tea\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"07d46c57dac7a4973cc6cc3d3f64632db468c7286120cc5c31c440075061aa3c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"d32366df33e5f54aeb4a5cac0b58617ef9795c05332cf16b4e3b9f7d2f335ba2","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:76]: failed to create customer resources with private key vault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001036920\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-private-kv-cluster-2xfb72\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-private-kv-cluster-2xfb72/operationStatuses/08584113914359747457\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-09/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-5dknb5dd5vlk/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-had2isw6vkgre\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:76]: failed to create customer resources with private key vault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001036920\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-private-kv-cluster-2xfb72\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-private-kv-cluster-2xfb72/operationStatuses/08584113914359747457\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-09/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-5dknb5dd5vlk/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-had2isw6vkgre\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"081521c798add1728bf279e017eb335673baca253c50f825ccdb18366411b8eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"3c01497d2687bfb43b9288af1b54486f516b205e9813d8dce28ff93101312372","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000d230b0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000d230b0\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"19e8a4598e6d15dd3debea08dccd0cc6aa98f928d064bac887f4468e23c6f7ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"49eac6d4a36a30efa10c1b8b1b8d2d65876ce6ea80106d791d42cf6c03bbbe8d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-21-49mv7b\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c1b0c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-21-49mv7b-blspsw\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-21-49mv7b-blspsw/operationStatuses/08584113913904847366\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-gkmbwrdjpztq/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qmbekpo5hsspo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-21-49mv7b\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c1b0c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-21-49mv7b-blspsw\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-21-49mv7b-blspsw/operationStatuses/08584113913904847366\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-gkmbwrdjpztq/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qmbekpo5hsspo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"24288ec26d4d3bd26d1f914e8db2b9f9ca4b2ed4b22c62e54cf451cc76aed2a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"a9db7c4a29702a102e7f5ecd1187fb046e212b54e149b8fd5e9d352e54ffbcd4","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-5-0-z6ssw4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f194e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-5-0-z6ssw4-vs425l\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-5-0-z6ssw4-vs425l/operationStatuses/08584113913637551351\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-8s727jgnk6zr/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-lwwwvssi7r4ue\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-10/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-5-0-z6ssw4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f194e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-5-0-z6ssw4-vs425l\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-5-0-z6ssw4-vs425l/operationStatuses/08584113913637551351\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-8s727jgnk6zr/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-lwwwvssi7r4ue\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-10/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"27f2b325501e7a713fe9437e9d0746a76519ce40bcc588d6b8e84bdf2eafe005","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"b49424c3e5a898d2542be3de8a944c781a1c54008483164cde2612601cc567b6","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:70]: failed to create customer resources for v20260901preview cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00079da80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-v20260901-2xzlq4\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-v20260901-2xzlq4/operationStatuses/08584113915367242561\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-5b2fj7lf5ptx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-xdteugpxef23k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:70]: failed to create customer resources for v20260901preview cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00079da80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-v20260901-2xzlq4\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-v20260901-2xzlq4/operationStatuses/08584113915367242561\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-5b2fj7lf5ptx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-xdteugpxef23k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"2bec5b91b2a4aa59363d8c4766601cadec18b99622746d9ef2b0f14bade8ffcd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"c3c4a6ccbac232171b2b65ce786cdfef92bbef99d093fd8e6cdd95fc35731f03","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:97]: failed to create customer resources for basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aa6d20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-cluster-rmgk9x\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-cluster-rmgk9x/operationStatuses/08584113915254253091\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-mcxckz7xhsq9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j3gkygvtvkvjs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-04/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:97]: failed to create customer resources for basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aa6d20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-cluster-rmgk9x\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-cluster-rmgk9x/operationStatuses/08584113915254253091\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-mcxckz7xhsq9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j3gkygvtvkvjs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-04/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"2ed90d5d8255604f294cd55a811491c8e6d2fc919846c4233e044dc499158b75","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"ca4be0c2537a5986ad337e9dd7b0c174dbbfd555b40b995690b3ea0bae599488","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:78]: failed to create customer resources for cilium CNI cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ad1980\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cilium-cl-xdnwwz\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cilium-cl-xdnwwz/operationStatuses/08584113915369084851\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-lrh4jzlp524b/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-dve5f7wncegny\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-11/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:78]: failed to create customer resources for cilium CNI cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ad1980\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cilium-cl-xdnwwz\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cilium-cl-xdnwwz/operationStatuses/08584113915369084851\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-lrh4jzlp524b/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-dve5f7wncegny\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-11/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"3156f851df31f91b4d1f92dfbe2df710b94593016b50ffe0ab075a83c3ec8d99","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"4b1f37cc80923b8d11e34699631a1958ecd1c54c87c846691565df3c140d290b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-20-kk49dv\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c0e960\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-20-kk49dv-srznwg\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-20-kk49dv-srznwg/operationStatuses/08584113913904019311\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-15/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-cggfsprbwr4j/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rksty5frww7dw\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-20-kk49dv\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c0e960\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-20-kk49dv-srznwg\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-20-kk49dv-srznwg/operationStatuses/08584113913904019311\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-15/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-cggfsprbwr4j/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rksty5frww7dw\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"324c297fe480083610618dc19666f723de512e83986f0bd9a51e6459870b85c6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"84c63d590be5b6794793c8c510fecef7965137dfae7568e99ea9e7ba5cffbfc9","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-23-8l7ngb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b1db60\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-23-8l7ngb-h5lrt2\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-23-8l7ngb-h5lrt2/operationStatuses/08584113913906278616\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-07/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-rfcmtkcs67tb/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ozfqahwps5hom\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-23-8l7ngb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b1db60\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-23-8l7ngb-h5lrt2\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-23-8l7ngb-h5lrt2/operationStatuses/08584113913906278616\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-07/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-rfcmtkcs67tb/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ozfqahwps5hom\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"33815bb2d09642c680f973dce26486ce1ae2f7c4c6f7c3a26d7fd2670a672f99","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"58e4dfcaa596cd88fda970ca6914db4000b0185d3538f68ac836493847042863","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:291]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dac1e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-noedge-s95n9h-cpl7fc\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-noedge-s95n9h-cpl7fc/operationStatuses/08584113912343166379\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-s95n9h-mm4b4h66xrsd/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rgyesixzav3z4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:291]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dac1e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-noedge-s95n9h-cpl7fc\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-noedge-s95n9h-cpl7fc/operationStatuses/08584113912343166379\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-s95n9h-mm4b4h66xrsd/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rgyesixzav3z4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"3c9e140f1020e227a56125a3977af8c3be796ec08905298ab49e9ec7ae018603","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"9d3778e9e95acd335ee5ab579b70317fa3b0f3a32b64bfc784b8aab8910d2702","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:80]: failed to create customer resources for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f84600\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cx-rg-hcp-cluster-279vtk\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cx-rg-hcp-cluster-279vtk/operationStatuses/08584113915372258288\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-x7gp5n7qjzxd/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fmxs36wu5sow2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-14/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:80]: failed to create customer resources for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f84600\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cx-rg-hcp-cluster-279vtk\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cx-rg-hcp-cluster-279vtk/operationStatuses/08584113915372258288\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-x7gp5n7qjzxd/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fmxs36wu5sow2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-14/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"3e85ebfde8f52df057ba2a0c3ee6d8a129c0fd3918045f1fd493d33698d281e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"493df6fcec5673d5971d28b2495e7b60a93b79fc07aeb785fb4063e1017f859d","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-22-cnb4hw\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c20820\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-22-cnb4hw-bwtj8b\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-22-cnb4hw-bwtj8b/operationStatuses/08584113913640193299\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-08/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-pfd2lgp68w4g/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-n3ogmkaqsinvk\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-22-cnb4hw\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c20820\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-22-cnb4hw-bwtj8b\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-22-cnb4hw-bwtj8b/operationStatuses/08584113913640193299\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-08/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-pfd2lgp68w4g/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-n3ogmkaqsinvk\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"4100a75446c05474c81d75fb4a6963270ff0224ec5b5dac18065376c13139209","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"0b7b3b363d12dcde6dbe7dcaf8d1bfff05c68c4872ac68b624b4b5dd24798a34","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:710]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014fe720\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-dg-minor-zdzkqx-qhfmmt\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-dg-minor-zdzkqx-qhfmmt/operationStatuses/08584113912216912001\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-zdzkqx-l22p2sw54pdw/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-m4n6shodaasxc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-06/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:710]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014fe720\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-dg-minor-zdzkqx-qhfmmt\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-dg-minor-zdzkqx-qhfmmt/operationStatuses/08584113912216912001\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-zdzkqx-l22p2sw54pdw/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-m4n6shodaasxc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-06/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"4281ae5d5f2e327448159ddd1aac6cffb40e1ec68e11c7a0f3b17664d5b21d35","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"8168feff532830edf3ac3497f6ce1e0b657479305b2853c1a48cbb283d165e76","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:77]: failed to create customer resources for private ingress cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bd0200\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-private-ingress-rjpgqd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-private-ingress-rjpgqd/operationStatuses/08584113915370331446\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-c4nx2jgq7mkh/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6rnsb3kbhjy5m\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-21/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:77]: failed to create customer resources for private ingress cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bd0200\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-private-ingress-rjpgqd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-private-ingress-rjpgqd/operationStatuses/08584113915370331446\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-c4nx2jgq7mkh/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6rnsb3kbhjy5m\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-21/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"478f9902aa790e19dfbff09fb0ab7d8f88e16b5bbe9463f36f2f5d76cc2eb6f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"02474ed7bffe3f2f3efb832206e4ee8bb2657d3eb50cde44d64f37e4e025c8d9","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:75]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000963640\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ephemeral-disk-nr8r8x\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ephemeral-disk-nr8r8x/operationStatuses/08584113912860313395\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-9jsgpbvbxf9r/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6xts4ycasve7q\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-05/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:75]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000963640\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ephemeral-disk-nr8r8x\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ephemeral-disk-nr8r8x/operationStatuses/08584113912860313395\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-9jsgpbvbxf9r/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6xts4ycasve7q\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-05/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"48a6149cfc5394a29327e312d0e2798e52e85ef76e0b0b015a98fc28dd1f6f42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"b27d8e1b8d937857f4f4398e505b9f826ce2b0dbdcdb3fdae66e91447431a872","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:93]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001cd1a80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-autoscale-cluster-59q7qk\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-autoscale-cluster-59q7qk/operationStatuses/08584113913180683547\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-23/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-9dxvcqrwjfqn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ssrwqrnhyyabs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:93]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001cd1a80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-autoscale-cluster-59q7qk\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-autoscale-cluster-59q7qk/operationStatuses/08584113913180683547\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-23/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-9dxvcqrwjfqn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ssrwqrnhyyabs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"50d9d3cc07e42755681edfb5a902dc1162b456c08a3f49f46bf076df0295239f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"378c2c4f5f003f5f14a5587c4fc6d847ccbc184c6e7fcf305a9df629726e8dce","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:74]: failed to create customer resources for patch-name cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00070ba60\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-patch-name-cluster-n55mjb\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-name-cluster-n55mjb/operationStatuses/08584113915366639693\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-22/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:74]: failed to create customer resources for patch-name cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00070ba60\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-patch-name-cluster-n55mjb\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-name-cluster-n55mjb/operationStatuses/08584113915366639693\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-22/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"51976b069c78b86eefbcb8bc5ac75154e425b195402783eb8ce2e782ba05e889","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"ebaee311f476e862bdc8a9ed9fb42966e18b1240f73ae004a1a47556a07ef340","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:218]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cf2a00\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-oidc-wi-cluster-qvs74t\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-oidc-wi-cluster-qvs74t/operationStatuses/08584113912059631916\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-m8svn75dghbq/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-s6kwp7xg43dbi\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-04/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:218]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cf2a00\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-oidc-wi-cluster-qvs74t\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-oidc-wi-cluster-qvs74t/operationStatuses/08584113912059631916\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-m8svn75dghbq/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-s6kwp7xg43dbi\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-04/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"5607f70647bf76e5e1d6ac4d4cd8b2230aea4003caea752b485b6b26fc45cde5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"303e71f56a7321ff7781f70fdf21e43e15eafba5a112fb795a421560bf8687d4","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:87]: failed to create customer resources for TLS endpoint cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010cf380\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-tls-endpoint-hcp-cluster-psgbhg\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-tls-endpoint-hcp-cluster-psgbhg/operationStatuses/08584113915373233860\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-15/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-2kkdh9d9gtp9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oo7pyvaeowk44\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:87]: failed to create customer resources for TLS endpoint cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010cf380\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-tls-endpoint-hcp-cluster-psgbhg\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-tls-endpoint-hcp-cluster-psgbhg/operationStatuses/08584113915373233860\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-15/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-2kkdh9d9gtp9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oo7pyvaeowk44\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"56d8599c3df3d48349a64fb62589725f0b59a8c16f9ee6cb9cb45ea94210ebf5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"d949d00c9bd448d0c2054b0f46eb2a81e5f314ec80d1542febb5b972b95b51fc","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:78]: failed to create customer resources for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001109320\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-list2xpmz4-575f9n\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-list2xpmz4-575f9n/operationStatuses/08584113913331185786\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-20/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-sq8bchrx6sgb/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3pwmnglx4zz7q\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:78]: failed to create customer resources for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001109320\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-list2xpmz4-575f9n\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-list2xpmz4-575f9n/operationStatuses/08584113913331185786\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-20/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-sq8bchrx6sgb/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3pwmnglx4zz7q\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"5750f0f1212adcf743cbe2e2e258c5c3776962a708222d0589e8e2a1941f1b2c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"cc5beb5e38150b98e6a39e86bd07ab29baf902b0225bf3a00a1113d79a08d467","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:283]: failed to create cluster customer resources for node-limit test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009ad040\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-node-limit-cluster-5wrht5\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-5wrht5/operationStatuses/08584113913007034136\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-12/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-hlk6qr5q8zl2/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:283]: failed to create cluster customer resources for node-limit test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009ad040\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-node-limit-cluster-5wrht5\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-5wrht5/operationStatuses/08584113913007034136\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-12/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-hlk6qr5q8zl2/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"5f74679805a99d3644c2a91066fd5ad8c61cf65f78a3c56515463c680b6e9f91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"b7d1a1e20d148974b57db9b0bf46b2d32a4fe75a72b18053adfcd3ffc10008fc","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:80]: failed to create cluster customer resources for cordon test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00089f100\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cordon-hcp-cluster-5jj9fq\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cordon-hcp-cluster-5jj9fq/operationStatuses/08584113912845247458\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-01/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-q5nkw446pf9v/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3wbuyr3ac6kzo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:80]: failed to create cluster customer resources for cordon test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00089f100\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cordon-hcp-cluster-5jj9fq\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cordon-hcp-cluster-5jj9fq/operationStatuses/08584113912845247458\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-01/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-q5nkw446pf9v/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3wbuyr3ac6kzo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"6cdfc2b46b4225201fd2985a3954ebbe3ae09bd1ee96684cc4225622e0902f2f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"5e81e681dd240666f7f73c328e1314295bc989aa39e4424d393b9191c79a5847","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:114]: failed to create customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b75c00\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ingress-cert-le-znhvzv\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ingress-cert-le-znhvzv/operationStatuses/08584113915367159662\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-4v9jkdchszqt/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-nhimtfs3fueps\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-23/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:114]: failed to create customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b75c00\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ingress-cert-le-znhvzv\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ingress-cert-le-znhvzv/operationStatuses/08584113915367159662\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-4v9jkdchszqt/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-nhimtfs3fueps\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-23/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"6ef75e57faeda4d7bc5c350dba5885d5ec62c46dc5e3c409793ed57e152ace0c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"f939e6af64ed35d589b26c26d6589830f2545bdd650fd5e4cd474c0aca8fe905","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:101]: failed to deploy dedicated managed identities\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010124e0\u003e: \n failed waiting for deployment \"mi-missing-mi-hcp-cluster-fgbmvd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-missing-mi-hcp-cluster-fgbmvd/operationStatuses/08584113915367121300\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-l28hcpvmt567/providers/Microsoft.Resources/deployments/clusterMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-l28hcpvmt567/providers/Microsoft.Resources/deployments/clusterNonMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:101]: failed to deploy dedicated managed identities\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010124e0\u003e: \n failed waiting for deployment \"mi-missing-mi-hcp-cluster-fgbmvd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-missing-mi-hcp-cluster-fgbmvd/operationStatuses/08584113915367121300\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-l28hcpvmt567/providers/Microsoft.Resources/deployments/clusterMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-l28hcpvmt567/providers/Microsoft.Resources/deployments/clusterNonMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"79679d26abb668141da4be7cc356775b3a02ee61e6b748c36c7d5a7bd4af97fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"f4c7d50db1341281f9bbc735079e7958656096dd31c71f4c0842b77c630f5f87","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:78]: failed to create customer resources for ARM64 nodepool cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00057ce00\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-arm64-vm-hcp-cluster-r5vvmz\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-arm64-vm-hcp-cluster-r5vvmz/operationStatuses/08584113915370321334\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-13/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-5rbptphj46kh/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6htyoibevj7ye\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:78]: failed to create customer resources for ARM64 nodepool cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00057ce00\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-arm64-vm-hcp-cluster-r5vvmz\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-arm64-vm-hcp-cluster-r5vvmz/operationStatuses/08584113915370321334\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-13/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-5rbptphj46kh/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6htyoibevj7ye\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"7adad705fbf7ad24e4a9f9317d08e742a0cbf836419ba85c3eddae9e2302c71f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"6efac1d441daa5ff54b4d89012d20f08600df5d158a441ec102e24943fc428e9","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:84]: failed to create customer resources for admin credential lifecycle cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001022100\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-admin-cred-lifecycle-5h9f9s-sdsp8q\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-admin-cred-lifecycle-5h9f9s-sdsp8q/operationStatuses/08584113915370097694\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-xlsqsjp7jqbr/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-yrpyalmshkqw2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-08/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:84]: failed to create customer resources for admin credential lifecycle cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001022100\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-admin-cred-lifecycle-5h9f9s-sdsp8q\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-admin-cred-lifecycle-5h9f9s-sdsp8q/operationStatuses/08584113915370097694\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-xlsqsjp7jqbr/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-yrpyalmshkqw2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-08/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"7cf1a22c519359fd957ad9e8a925fc4d91cb479df948cf235f3b153eafd0ebf7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"bfb111b44b68e6d49553ea155dabf55537d9068f181b21a0605c2d49bbf1a5f0","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:77]: failed to create customer resources for private KAS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001129640\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-private-kas-dfpc6t\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-private-kas-dfpc6t/operationStatuses/08584113915370549584\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-dddkkc8rw76v/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-jxdbnndsp5js2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:77]: failed to create customer resources for private KAS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001129640\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-private-kas-dfpc6t\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-private-kas-dfpc6t/operationStatuses/08584113915370549584\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-dddkkc8rw76v/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-jxdbnndsp5js2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"87c84aa6bdeca1e33cfb0d172f2012a5b1d1d6418e0732350a1609305c7d38bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"29363b364d248244b8cab05224e499f828ca4b3558f6b271121ac89221e31f80","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:90]: failed to create customer resources for IDMS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000686ec0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-idms-e2e-hcp-cluster-54nj9g\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-idms-e2e-hcp-cluster-54nj9g/operationStatuses/08584113913114169636\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-pwtsd6dd5nm5/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-lv7dvzspd2e5i\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-21/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:90]: failed to create customer resources for IDMS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000686ec0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-idms-e2e-hcp-cluster-54nj9g\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-idms-e2e-hcp-cluster-54nj9g/operationStatuses/08584113913114169636\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-pwtsd6dd5nm5/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-lv7dvzspd2e5i\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-21/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"888830aaa4738527a9bfebf5945d2808e79f3c8a3f7d6a07f88fcf73665bacde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"eb699857478bc226c6dc8aaf326a1110ec3eb808736fd153bc787c847ef8fd08","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-21-tlkfpt\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012d42e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-21-tlkfpt-dgcfjk\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-21-tlkfpt-dgcfjk/operationStatuses/08584113913452272222\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-14/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-vfkmrmxhjl5d/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-gwvc3vdjzhp2q\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-21-tlkfpt\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012d42e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-21-tlkfpt-dgcfjk\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-21-tlkfpt-dgcfjk/operationStatuses/08584113913452272222\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-14/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-vfkmrmxhjl5d/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-gwvc3vdjzhp2q\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"8969d8bad205b41d0dcee8d5550ae4ac2b377ea8801e975df74a33cfbe4631bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"8b9341db534e17f1c007a11f07836dbec6a14166f62a28d78b3b65efe63e43a7","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0009be4e0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-8x9trr4ppbcc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0009be4e0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-8x9trr4ppbcc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8fd8d7672c079bea041cf151014620d8c24131aade13575d81989cef9ee57be8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"29738fe680afcd7bb868f6b4e7bf0e48f78b9de7c821820e7ce46f1b77273631","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:104]: failed to create customer resources for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bfe780\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-pullsecret-hcp-cluster-qzdltq\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-pullsecret-hcp-cluster-qzdltq/operationStatuses/08584113915375639922\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-16/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-ckn8x8pc56zb/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-p2h32vyw2l2ma\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:104]: failed to create customer resources for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bfe780\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-pullsecret-hcp-cluster-qzdltq\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-pullsecret-hcp-cluster-qzdltq/operationStatuses/08584113915375639922\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-16/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-ckn8x8pc56zb/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-p2h32vyw2l2ma\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"91b0ffecfd6854d0c4992b76d7f21482be67ec782222a0c26a30ddbc466cf957","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"c80bf9057eb592b0d78f55fa548e2f6ce83aaf5494a5e05657657b2e3f4c11d9","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:130]: failed to create customer resources for aggregated feature cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010b9ba0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-agg-cluster-fcg4sb\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-agg-cluster-fcg4sb/operationStatuses/08584113914362490457\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-tk7ckttkvc2h/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-zkzqbkxbrvnhu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-00/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:130]: failed to create customer resources for aggregated feature cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010b9ba0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-agg-cluster-fcg4sb\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-agg-cluster-fcg4sb/operationStatuses/08584113914362490457\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-tk7ckttkvc2h/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-zkzqbkxbrvnhu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-00/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"983dba0f9a5517e1ce8031a1b4d83cece1312c5466c434021c7af6f37690c02c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"a6fdde63eb30cf8fde25cd7079bb5bd9247a18fcc4fc428d6930daff70e0c43f","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:268]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bdb9e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-shoebox-hcp-cluster-65ml2z\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-shoebox-hcp-cluster-65ml2z/operationStatuses/08584113912169288703\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-ltfr4g8jwwws/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-22eudp3sazqeq\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-08/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:268]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bdb9e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-shoebox-hcp-cluster-65ml2z\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-shoebox-hcp-cluster-65ml2z/operationStatuses/08584113912169288703\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-ltfr4g8jwwws/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-22eudp3sazqeq\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-08/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"a024a038f35cb1f02d499b3329e0dbfc2867ff1e302f0d8a58b610ab81e5bb05","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"21af537e81ae2811dae2782f1541dea999e6b6e005eb5ba0438334b0110de505","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:151]: failed to create customer resources for patch-tags cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eb5680\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-patch-tags-cluster-wshhfl\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-tags-cluster-wshhfl/operationStatuses/08584113913827373414\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-jcg4mtdpvc69/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fklo75htpxzxc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-13/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:151]: failed to create customer resources for patch-tags cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eb5680\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-patch-tags-cluster-wshhfl\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-tags-cluster-wshhfl/operationStatuses/08584113913827373414\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-jcg4mtdpvc69/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fklo75htpxzxc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-13/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"a1e64f4ff56c6cb4643b06d472c79c09857c341a2da94544513cb48b0fe73b27","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"dc85a4e926dccafcd7a84d59a6cd891fae237d24dda05b7aef3fa9e97dec9da2","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:79]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d98940\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-disabled-image-registry-hcp-cluster-kjxtrg\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-disabled-image-registry-hcp-cluster-kjxtrg/operationStatuses/08584113913314624478\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-22/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-w7nl4t2fw4sl/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qmh74zogaodhg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:79]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d98940\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-disabled-image-registry-hcp-cluster-kjxtrg\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-disabled-image-registry-hcp-cluster-kjxtrg/operationStatuses/08584113913314624478\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-22/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-w7nl4t2fw4sl/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qmh74zogaodhg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"a351cd4959c20474bcc13e0024114bbbcee0a7c9e86bf0e589f7b7a8f1789ef5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"004b42a6843d9c599be4103c0b3a7bbc53c9a341d8d2c16ff587dddcdca93fea","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:71]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001080a20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-delete-hcp-cluster-wjhgxf\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-delete-hcp-cluster-wjhgxf/operationStatuses/08584113912929453892\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-779f92ndzf75/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rcpnvqrcxdqmo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-09/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:71]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001080a20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-delete-hcp-cluster-wjhgxf\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-delete-hcp-cluster-wjhgxf/operationStatuses/08584113912929453892\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-779f92ndzf75/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rcpnvqrcxdqmo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-09/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"a8a51c354323e503eea7270364f42f2e8c86fb49c194a470a1dc0f44107cf939","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"b9b552cfce8aa134690435adce391de22a7d10909a7f0b0b344d1de5d5f13741","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:71]: failed to create customer resources for cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ba70c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-hcp-cluster-np-128-v7nvxr\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-hcp-cluster-np-128-v7nvxr/operationStatuses/08584113915365585990\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-06/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d7wljqzdp555/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3ljyjjx2n7gxo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:71]: failed to create customer resources for cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ba70c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-hcp-cluster-np-128-v7nvxr\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-hcp-cluster-np-128-v7nvxr/operationStatuses/08584113915365585990\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-06/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d7wljqzdp555/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3ljyjjx2n7gxo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"b28d01e87112519270e6e11da26d47a5909d84e514b5dc272e9668110f1de1aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"df7c02235ae5a45c83397b743a331f854e72bcbbb98b42830177e7f9d038928e","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:84]: failed to create customer resources with private key vault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006e6620\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cilium-cluster-nf72s7\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cilium-cluster-nf72s7/operationStatuses/08584113914453431415\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-01/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-stcwrwm7gdff/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-4vnk4izirdfzu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:84]: failed to create customer resources with private key vault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006e6620\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cilium-cluster-nf72s7\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cilium-cluster-nf72s7/operationStatuses/08584113914453431415\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-01/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-stcwrwm7gdff/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-4vnk4izirdfzu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"b4cf9eff92f69ced7e1e9b37ea4276ff9df14bcbc9c114a7710d6d1335496ff1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"590a7ce9c34dea2488d7bcaa818aa04f1e7333c2c32aa265eb01f529e4c09168","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-5-0-4b8xtf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00088c740\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-5-0-4b8xtf-v47dt6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-5-0-4b8xtf-v47dt6/operationStatuses/08584113913751466362\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-n8kzhs252rzq/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oowsf3asvtt4c\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-06/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-5-0-4b8xtf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00088c740\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-5-0-4b8xtf-v47dt6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-5-0-4b8xtf-v47dt6/operationStatuses/08584113913751466362\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-n8kzhs252rzq/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oowsf3asvtt4c\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-06/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"b7c303e082a673e1e03a920af28a090172f877130fd992b067ddd07ecd360ce0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"1353cd7fc2e7eb95af2a2957b1ec2d2b9416cd1f12ca6cfeab46ea5f033a2da7","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c05160\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-zsckxd6r8cnx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-zsckxd6r8cnx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c05160\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-zsckxd6r8cnx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-zsckxd6r8cnx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"bb14ae84952a406610caf29bd8584b6cf2e8a89f4c8644bac400fdf762266800","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"ac85942acdcf9dd62304e20814465a73dab2c6546d5dfd1ec23a0709cce01819","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:88]: failed to create customer resources for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001012880\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cidr-connectivity-test-lq9dng\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-lq9dng/operationStatuses/08584113915355806276\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-10/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-ncn82bv9lpkb/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:88]: failed to create customer resources for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001012880\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cidr-connectivity-test-lq9dng\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-lq9dng/operationStatuses/08584113915355806276\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-10/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-ncn82bv9lpkb/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"bb71bc5ef8a9bbe7ae71eea7fe7d84c4c7f24fb5cb6096e44323aba011a525e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"63857a171b9f09d8d921021f794a90f8ab826e76cf935d7a69f3776cf78381b3","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:121]: failed to deploy managed identities for back-level version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019efc0\u003e: \n failed waiting for deployment \"mi-cluster-ver-4-20-b2znhh\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-ver-4-20-b2znhh/operationStatuses/08584113913906857154\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-wmxx2c8c9twx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-11/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:121]: failed to deploy managed identities for back-level version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019efc0\u003e: \n failed waiting for deployment \"mi-cluster-ver-4-20-b2znhh\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-ver-4-20-b2znhh/operationStatuses/08584113913906857154\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-wmxx2c8c9twx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-11/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c23affd5bcb5c5695fe9ec861808a8ee83889e7fd42a8e5651470217652c83f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"ed72ed9221365984a17f27e8ed0e35e5508168087855a2c0749774e962bc35bc","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-22-6fxxn7\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d68380\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-22-6fxxn7-ng8rb6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-22-6fxxn7-ng8rb6/operationStatuses/08584113913906446639\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-j8b5hvpzhn5m/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-7frcmwevtsxko\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-22-6fxxn7\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d68380\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-22-6fxxn7-ng8rb6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-22-6fxxn7-ng8rb6/operationStatuses/08584113913906446639\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-j8b5hvpzhn5m/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-7frcmwevtsxko\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-02/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c277dcc880608051522b40f64ea81f55fcd3c67256565c0b8468a4badf64111c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"a4a7b00e0cf6167aca09a6c39407324ab2cbdf83285b14e600ca3cbebdae2762","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:570]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c78700\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-downgrade-hfsp4j-mttxxb\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-downgrade-hfsp4j-mttxxb/operationStatuses/08584113912282350438\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-hfsp4j-w9pdctvxrqv9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-7uiwfmbeixp3i\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:570]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c78700\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-downgrade-hfsp4j-mttxxb\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-downgrade-hfsp4j-mttxxb/operationStatuses/08584113912282350438\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-hfsp4j-w9pdctvxrqv9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-7uiwfmbeixp3i\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c33bcbe6998a085eda326f182bb4fa941dc283930ada794fc85601df2ba13b32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"6d4c6f30d7a0118a761afb53daa3e256cfba084c934cfa3569d0253ab1d5a261","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:85]: failed to create customer resources for external auth cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000baaf40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-cluster-845ctr\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-cluster-845ctr/operationStatuses/08584113913017973521\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-28pr5rtrjz7f/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ltdphft7fgrwg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-19/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:85]: failed to create customer resources for external auth cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000baaf40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-cluster-845ctr\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-cluster-845ctr/operationStatuses/08584113913017973521\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-28pr5rtrjz7f/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ltdphft7fgrwg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-19/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c5b4cdff897e009233f664a3ccabbc3a9d9af371c5168c0d8efc7b4779dd936b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"0a134035c835ada31f01131908200d3c87d62d1fac56b860bc5699a1bf6f2546","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:83]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c96200\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-negative-tests-cluster-pqtw6g\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-pqtw6g/operationStatuses/08584113912154838800\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-16/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-qhmbvhnrg56r/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:83]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c96200\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-negative-tests-cluster-pqtw6g\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-pqtw6g/operationStatuses/08584113912154838800\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-16/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-qhmbvhnrg56r/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c65105a172043bec2410ef35d2273ff68fbbbc9339eb2d11abc0b9cc292aedda","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"8254913197285306ce0204858e98e4a4547494803895048d378c8049c527c44a","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-5-0-p8mv48\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce0ae0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-5-0-p8mv48-rr8c45\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-5-0-p8mv48-rr8c45/operationStatuses/08584113913442668481\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-18/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-qmng4jgqjs56/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6i3b5eapw7ct4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-5-0-p8mv48\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce0ae0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-5-0-p8mv48-rr8c45\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-5-0-p8mv48-rr8c45/operationStatuses/08584113913442668481\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-18/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-qmng4jgqjs56/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6i3b5eapw7ct4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c74bbbff1ef75043b7ae445448ff485d3663f17a893c1429496a2bd4affd0692","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"5275008bf6e2cd955a92ff0c773682149b95d29566406f66e1fed2199df22e64","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-22-t7jcbl\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00071d980\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-22-t7jcbl-8d776t\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-22-t7jcbl-8d776t/operationStatuses/08584113913541685590\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-9z48wmwt6pmp/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-zmpandh64xdog\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-16/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-22-t7jcbl\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00071d980\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-22-t7jcbl-8d776t\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-22-t7jcbl-8d776t/operationStatuses/08584113913541685590\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-9z48wmwt6pmp/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-zmpandh64xdog\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-16/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c902ecf219bd9cc7e05bfb81ad29881278cf3717fd0f67636e423f180473184c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"b60748e01d3770081b9092311833c353d24a585d648763d5987389842fb13c7f","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-20-gc4q5q\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009d6c00\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-20-gc4q5q-z24rh4\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-gc4q5q-z24rh4/operationStatuses/08584113913639830680\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-s5dpm7kj5krx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-76i3fdasycsia\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-04/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-20-gc4q5q\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009d6c00\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-20-gc4q5q-z24rh4\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-gc4q5q-z24rh4/operationStatuses/08584113913639830680\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-s5dpm7kj5krx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-76i3fdasycsia\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-04/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"ceefb415d5d8396f3948093afc22d335c21f420b30f5715b41f512ef2c497fd6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"de91715b5519c3e9c48e9f9db31ee57baa410e66aff582268d9803787a051a40","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:72]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa4b80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-fips-enabled-cluster-zm2g24\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-fips-enabled-cluster-zm2g24/operationStatuses/08584113915371376474\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-07/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-wckv5dm6jq8x/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3h7v3eeeqgjeu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:72]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa4b80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-fips-enabled-cluster-zm2g24\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-fips-enabled-cluster-zm2g24/operationStatuses/08584113915371376474\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-07/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-wckv5dm6jq8x/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3h7v3eeeqgjeu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"d37c007fe5650b6dbcaf0dc9e8bf67771d55d192c221036180c5686d87c18c82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"b0ee5240012c5afec47ab2c288045073ea38119a5b50f900847fb95919697b1a","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-23-dxtnkh\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a4e9e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-23-dxtnkh-qg2zth\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-23-dxtnkh-qg2zth/operationStatuses/08584113913448474837\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-17/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-zcw9r52tsr2b/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-f5hs6ydcv2ewe\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-23-dxtnkh\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a4e9e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cp-ystream-upgrade-4-23-dxtnkh-qg2zth\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cp-ystream-upgrade-4-23-dxtnkh-qg2zth/operationStatuses/08584113913448474837\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-17/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-zcw9r52tsr2b/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-f5hs6ydcv2ewe\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"dcc37480b614822ab72bcc2e3e1bdf03cd9f3975610c4c03a2f4f1234bc59528","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"c3a66ee84a59916e6ac4929f00304fe7deb56ae962c7df382745b1e42339ff55","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-21-jz52ld\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001036ec0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-21-jz52ld-fx8wn6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-jz52ld-fx8wn6/operationStatuses/08584113913641594984\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-fpmccnv5bjnp/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qdnzuabix4bjc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-03/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-21-jz52ld\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001036ec0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-21-jz52ld-fx8wn6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-jz52ld-fx8wn6/operationStatuses/08584113913641594984\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-fpmccnv5bjnp/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qdnzuabix4bjc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-03/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"e1872b8f764e71bd2009b187739a245541860bcb52a7f3f498515b9e0e6400f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"63ff74a61a4695146a80e07d486ea919d166ac0907ae451cd218aaed86a6db14","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f30a60\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-kgpnd2-45phqx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-kgpnd2-45phqx/operationStatuses/08584113912586089579\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-07/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-kgpnd2-t5lrjpxgmgpw/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-igjsnmdrgrbxc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f30a60\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-kgpnd2-45phqx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-kgpnd2-45phqx/operationStatuses/08584113912586089579\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-07/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-kgpnd2-t5lrjpxgmgpw/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-igjsnmdrgrbxc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"ea83e1a09bdfd77846c0391b44fbf8dc6ae40a76a9c5a9ddba05d0dd25941b79","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"6664222cdd56fe98e8670b4874f17c78c7d2168808f807394b24f384709545e8","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00128ea20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-dthbtj-dqjtmg\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-dthbtj-dqjtmg/operationStatuses/08584113912431610183\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-15/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-dthbtj-7tbfh95dw7tz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wzdmuce4v2olu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00128ea20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-dthbtj-dqjtmg\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-dthbtj-dqjtmg/operationStatuses/08584113912431610183\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-15/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-dthbtj-7tbfh95dw7tz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wzdmuce4v2olu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"eb478cbd4bbae0bac3961d95b2d34f86e2afd36dcd5adc3973e103f52c8036eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"f473bdd0d73590343184ba03445fe9c69d83d507ba5dec3b6434fd8b6dd23c83","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:73]: failed to create customer resources for v20261001preview cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000826c60\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-v20261001-4sckh4\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-v20261001-4sckh4/operationStatuses/08584113915372800941\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-r6vgswb52dpb/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-7e767rdphst3w\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-17/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:73]: failed to create customer resources for v20261001preview cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000826c60\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-v20261001-4sckh4\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-v20261001-4sckh4/operationStatuses/08584113915372800941\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-r6vgswb52dpb/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-7e767rdphst3w\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-17/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"f28332bab65c882d48f39d79ee5b807411f4b6d4a51e9a560e2a9f92d439a59c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"d71fff82c2dfe656568e32c9670f076351175cfc25591cc171d7af2c71095862","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:81]: failed to create customer resources for first cluster \"basic-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009e8f80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster-kq9qb6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-kq9qb6/operationStatuses/08584113915366174777\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-18/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-qvskb9lgr2hn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eqbgltuyzhpg4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:81]: failed to create customer resources for first cluster \"basic-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009e8f80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster-kq9qb6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-kq9qb6/operationStatuses/08584113915366174777\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-18/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-qvskb9lgr2hn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eqbgltuyzhpg4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"f84a596ae94fe95c04053d4ab2c58a019cd9399be4c993e8e0e385cc0a62affc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"292ecbb651b87ea834cc7055016245c2d8aa052ab0fd931987019f543f6e4d6e","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:431]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001402320\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-skip-minor-l4vpc6-lszvf9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-skip-minor-l4vpc6-lszvf9/operationStatuses/08584113912334810014\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-13/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-l4vpc6-mxzhwk4xdm5z/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oajio3ejrfg24\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:431]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001402320\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-skip-minor-l4vpc6-lszvf9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-skip-minor-l4vpc6-lszvf9/operationStatuses/08584113912334810014\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-13/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-l4vpc6-mxzhwk4xdm5z/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oajio3ejrfg24\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"facc368cc28f49bb6f7ea007df97528e58dd73bc261f4b3c8ac947344d23cdea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"93efcc50d250cefb160b3d3164d61008ab5df04f873fdfcbbf699d728ee56461","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:82]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ed09c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-np-labels-taints-2jzh8d\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-np-labels-taints-2jzh8d/operationStatuses/08584113912724860835\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-ftnsr25mhp4v/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-r6cuakilyx67k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-00/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:82]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ed09c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-np-labels-taints-2jzh8d\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-np-labels-taints-2jzh8d/operationStatuses/08584113912724860835\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-ftnsr25mhp4v/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-r6cuakilyx67k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-00/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":59,"failure_pattern_ids":["20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54"],"matched_occurrences":59,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}