{"meta":{"date":"2026-06-30","timezone":"UTC","generated_at":"2026-08-28T05:17:51Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":5,"failed_runs":1,"runs_with_occurrences":1,"runs_with_failure_pattern_matches":1,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":4,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072036288210407424","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-06-30T19:15:22Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071986026833776640","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-06-30T15:55:39Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-30T11:44:43Z"},"failed_at":["e2e"],"failed_test_count":43,"occurrences":[{"row_id":"051b8d660935100cf00b9a2bc40fafb2588b27954ebe66bfe2fc629028ea9e0a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"34be6c410ebea8f42dd1fb2a969da344a6e92049f925aaf8aeb095d79883b63e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:95]: failed to create customer resources for cluster \"cluster-candidate-4-21-snpq9s\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c8c9a0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-21-snpq9s-jlrh9l\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-21-snpq9s-jlrh9l/operationStatuses/08584187856226833932\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-5vdsr5/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-uedd433jn6rrw\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:95]: failed to create customer resources for cluster \"cluster-candidate-4-21-snpq9s\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c8c9a0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-21-snpq9s-jlrh9l\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-21-snpq9s-jlrh9l/operationStatuses/08584187856226833932\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-5vdsr5/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-uedd433jn6rrw\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"062c00ce282a2cd31c2d3b50260499254d2002ece012adb3c6bac4e67864cc61","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"cae70c071af5d80d4dd2b47aff48a145a85ba023612468bcb95c49586bec0bda","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:89]: failed to create customer resources for IDMS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a50e20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-idms-e2e-hcp-cluster-vhb966\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-idms-e2e-hcp-cluster-vhb966/operationStatuses/08584187856875903212\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-d2rhb9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-lv7dvzspd2e5i\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:89]: failed to create customer resources for IDMS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a50e20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-idms-e2e-hcp-cluster-vhb966\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-idms-e2e-hcp-cluster-vhb966/operationStatuses/08584187856875903212\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-d2rhb9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-lv7dvzspd2e5i\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"126ca341a49ef3f42b76f5ee9de58bfec5f71d47469cedeb4b50b70cf47303fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"9ccc60807bb3d0f8b59a971d84e36cb511ac554bfb0ecafd693050914f979d3e","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:122]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cfabc0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-tfkh4v-kl9s9t\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-tfkh4v-kl9s9t/operationStatuses/08584187856742704984\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-tfkh4v-r7jp9r/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-d6kl6v5o766mm\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:122]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cfabc0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-tfkh4v-kl9s9t\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-tfkh4v-kl9s9t/operationStatuses/08584187856742704984\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-tfkh4v-r7jp9r/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-d6kl6v5o766mm\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"12bd03e3b22c04118efb7d34cbc66c81093c3fc9e1b2480596d21f06e211046b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"bcc21619c009b761b74efa81cefb7f2dae7ecf0fb9430884ab2d08b2f5359da7","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:579]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e08ca0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-downgrade-w5r6p4-thj4kq\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-downgrade-w5r6p4-thj4kq/operationStatuses/08584187855779410769\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-w5r6p4-cqjc9p/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-li3mbgo574w7y\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:579]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e08ca0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-downgrade-w5r6p4-thj4kq\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-downgrade-w5r6p4-thj4kq/operationStatuses/08584187855779410769\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-w5r6p4-cqjc9p/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-li3mbgo574w7y\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"155a60ad37b4dd51b1e5ef0700e918ff0ce1b00a1d5be685dca40a998c61afe7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"500d13171fa535f3ee3716a480691bb574ee03c1626eec39d2336dd608b99b86","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:207]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010740c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-oidc-wi-cluster-g8tpd4\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-oidc-wi-cluster-g8tpd4/operationStatuses/08584187855657382633\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gt4j9v/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-s6kwp7xg43dbi\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:207]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010740c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-oidc-wi-cluster-g8tpd4\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-oidc-wi-cluster-g8tpd4/operationStatuses/08584187855657382633\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-gt4j9v/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-s6kwp7xg43dbi\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"195e6201ba71152e2f79660039fdefb1afadd2886bde9441d5f02c0b97bb58d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"273823a019f7c9d93136bbebc8def0ef3e38de316e9a1602cbe8671a78eac958","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:84]: failed to create customer resources for external auth cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009cac60\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-cluster-cvvms6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-cluster-cvvms6/operationStatuses/08584187856875515968\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-9g6rx9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ltdphft7fgrwg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:84]: failed to create customer resources for external auth cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009cac60\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-cluster-cvvms6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-cluster-cvvms6/operationStatuses/08584187856875515968\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-9g6rx9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ltdphft7fgrwg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"1c4d7683b9c90e7ca2e79c7e085ef9a250d52a81386e534e15984191bc7c57e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"a35a4da4a7237e28b701f0b99f0eecdd000ba02a38c029b5c2d2fa6da870fdd6","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:79]: failed to create customer resources for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006ee960\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cx-rg-hcp-cluster-h9tz2c\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cx-rg-hcp-cluster-h9tz2c/operationStatuses/08584187858039535675\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-wwvf8k/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fmxs36wu5sow2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:79]: failed to create customer resources for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006ee960\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cx-rg-hcp-cluster-h9tz2c\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cx-rg-hcp-cluster-h9tz2c/operationStatuses/08584187858039535675\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-wwvf8k/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fmxs36wu5sow2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"2080e38120eeef5fd4b4b3f4409a1aef58d5fb3bea5cefd9109968051f12f208","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"f1cc88e2e1050bbe52cb39e7358522c6a4266e2a4f3b789c3d67dfbd74510560","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:92]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-19-j86mr5\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dab380\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-19-j86mr5-h8dzxs\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-j86mr5-h8dzxs/operationStatuses/08584187855615521778\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-8bcxrs/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-tzhrgbwizlkbk\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:92]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-19-j86mr5\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dab380\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-19-j86mr5-h8dzxs\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-j86mr5-h8dzxs/operationStatuses/08584187855615521778\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-8bcxrs/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-tzhrgbwizlkbk\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"2124aad5e55d420b7df450b78836ca94b5b26df13810c68a46c4c0e8c7d431e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"6f1e97cf86a258eea5ba652d10722e0ae8cb510308dde083aa0ecaf21bfc5680","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:97]: failed to create customer resources for basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00060d580\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-cluster-hdl2hm\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-cluster-hdl2hm/operationStatuses/08584187857936124773\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-7pfrcv/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j3gkygvtvkvjs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:97]: failed to create customer resources for basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00060d580\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-cluster-hdl2hm\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-cluster-hdl2hm/operationStatuses/08584187857936124773\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-7pfrcv/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j3gkygvtvkvjs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"3495f9c0356e8cd9a6d04e0e1016b31feeacd0e44e40590c010602cca23bd305","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"77b40d648dced3d8e5d3fbbc5b5eb87c7f24b138a5b470586495c840e5a2f82d","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:117]: failed to deploy managed identities for back-level version 4.19\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013808e0\u003e: \n failed waiting for deployment \"mi-cluster-ver-4-19-zk7b56\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-ver-4-19-zk7b56/operationStatuses/08584187858039921366\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-8grkk8/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:117]: failed to deploy managed identities for back-level version 4.19\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013808e0\u003e: \n failed waiting for deployment \"mi-cluster-ver-4-19-zk7b56\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-ver-4-19-zk7b56/operationStatuses/08584187858039921366\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-8grkk8/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"367d4383d55414faf1feda08903bcc7ca1b0e442b8b2c6dd6db22e00763fbafc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"a83201a94a704c00ee7627b06a1da25132db69b4968b624673a7b64ba3520b99","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:102]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000620540\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ephemeral-disk-2k6k98\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ephemeral-disk-2k6k98/operationStatuses/08584187856873136686\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-zrh7v9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6xts4ycasve7q\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:102]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000620540\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ephemeral-disk-2k6k98\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ephemeral-disk-2k6k98/operationStatuses/08584187856873136686\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-zrh7v9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6xts4ycasve7q\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"4a64e0c3be060ffb0083b1ca16721954e992e049d2390cea164bfec4d3d7e9bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"a57e91da831d03a19b0c0cc92579b0175463252c0a61fb07ec0c35f9996cb865","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:73]: failed to create customer resources for cilium CNI cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010aecc0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cilium-cl-fxw9ws\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cilium-cl-fxw9ws/operationStatuses/08584187857935123531\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-zgvs8z/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-dve5f7wncegny\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:73]: failed to create customer resources for cilium CNI cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010aecc0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cilium-cl-fxw9ws\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cilium-cl-fxw9ws/operationStatuses/08584187857935123531\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-zgvs8z/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-dve5f7wncegny\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"4af53f4959eaf7a64cd59f9cfe4df5f1a07373350b9de36261a8f99e6fa149da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"7424c4f23072dd7f3e87770531a92600cee18a30634b64d9910f9b55929d3fe5","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:77]: failed to create customer resources for ARM64 nodepool cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f093a0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-arm64-vm-hcp-cluster-snvg8v\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-arm64-vm-hcp-cluster-snvg8v/operationStatuses/08584187858035517963\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-5gslgw/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6htyoibevj7ye\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:77]: failed to create customer resources for ARM64 nodepool cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f093a0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-arm64-vm-hcp-cluster-snvg8v\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-arm64-vm-hcp-cluster-snvg8v/operationStatuses/08584187858035517963\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-5gslgw/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-6htyoibevj7ye\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"59935a7dfb867bff69b8d3be144bf87d81b64cf63645c2b89e34ebd89adf7116","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"77f1b7824560703925a292a7041cdf47ad3a12c83a025c97a7c65a17fa472039","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:95]: failed to create customer resources for cluster \"cluster-candidate-4-20-8wbp2c\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0f220\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-20-8wbp2c-m72l5v\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-20-8wbp2c-m72l5v/operationStatuses/08584187855622714364\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-vcmx9c/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-yho4o5rtioo24\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:95]: failed to create customer resources for cluster \"cluster-candidate-4-20-8wbp2c\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0f220\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-20-8wbp2c-m72l5v\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-20-8wbp2c-m72l5v/operationStatuses/08584187855622714364\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-vcmx9c/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-yho4o5rtioo24\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"5c84e84c8828962402f94e5e98bdad520754e224ca99caf241edd5d1fe18479b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"1b5e4b2b6c38660650be734f1ec5e7c9be92da2a62258a329a69870223072e3b","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:103]: failed to create customer resources for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00092d060\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-pullsecret-hcp-cluster-h6xfln\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-pullsecret-hcp-cluster-h6xfln/operationStatuses/08584187857890138763\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-2lsrl6/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-p2h32vyw2l2ma\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:103]: failed to create customer resources for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00092d060\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-pullsecret-hcp-cluster-h6xfln\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-pullsecret-hcp-cluster-h6xfln/operationStatuses/08584187857890138763\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-2lsrl6/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-p2h32vyw2l2ma\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"5e5fa5ee50e6a93f5af2c0a5193b286ab40087a651bf312f960b7003e65d0bae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"f2f35e2f8b26e8d5fbf2b68c98c1d4ffa92106a6be0d5cc595477381dee1c8d4","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:90]: failed to create customer resources for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00080bbc0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cidr-connectivity-test-9wrxj4\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-9wrxj4/operationStatuses/08584187858033619638\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-75rvdf/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:90]: failed to create customer resources for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00080bbc0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cidr-connectivity-test-9wrxj4\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cidr-connectivity-test-9wrxj4/operationStatuses/08584187858033619638\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-75rvdf/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wihoigd7cpguy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"6406e02a716971bda4d3404d3367b7521be36fd4ae6302bd97b6ccd4775dacdc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"ea6a6cd90ad562eb3c02d92f54fb70dbc35b9e003f1ad7afcf96d803187301b5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:95]: failed to create customer resources for cluster \"cluster-candidate-4-22-vq6vk4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d10940\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-22-vq6vk4-5tjqb7\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-22-vq6vk4-5tjqb7/operationStatuses/08584187856828514260\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-q8wmpl/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-pfswajy6jao2s\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:95]: failed to create customer resources for cluster \"cluster-candidate-4-22-vq6vk4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d10940\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-4-22-vq6vk4-5tjqb7\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-4-22-vq6vk4-5tjqb7/operationStatuses/08584187856828514260\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-q8wmpl/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-pfswajy6jao2s\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"671e319ac765838b1aeec8512cc200ab118262f167890a7749a081d00bf51ea2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"98d5703ec40c112dd994dbb997c832fbbcca1673f291db6c3f5045ebbfcbd9d3","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:81]: failed to create customer resources for first cluster \"basic-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d07940\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster-znkv24\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-znkv24/operationStatuses/08584187858039738005\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-5cstzv/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eqbgltuyzhpg4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:81]: failed to create customer resources for first cluster \"basic-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d07940\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster-znkv24\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-znkv24/operationStatuses/08584187858039738005\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-5cstzv/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eqbgltuyzhpg4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"699b443f240221d4a4b6281119fbca59f8e3160d73727930e06a027e0a6e5a20","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"f9b971e49b61dfee9197525ff123c2091e020c73e8f671a4243dac9f8077c16b","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:83]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d79000\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-negative-tests-cluster-h4sd57\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-h4sd57/operationStatuses/08584187855739788076\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-5fjq8b/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:83]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d79000\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-negative-tests-cluster-h4sd57\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-negative-tests-cluster-h4sd57/operationStatuses/08584187855739788076\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-5fjq8b/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oeayaalhj7wjy\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"6a4a40dc871c0cfa4927340328702c7ddc0dca6d7a06fd1baf0e1bd8ac4b7851","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"d46662b6c5c3b5077517422943c458d95651ea56cb570be219e9b65d5e7778f5","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:77]: failed to create customer resources for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b84b20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-listhq4n6c-wzns28\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-listhq4n6c-wzns28/operationStatuses/08584187856776557038\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-9ltlvd/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-my6owol6p5vmm\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-02' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-02' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-02' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-02' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-02' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-02' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-02' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-02' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-02' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-02' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:77]: failed to create customer resources for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b84b20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ea-listhq4n6c-wzns28\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ea-listhq4n6c-wzns28/operationStatuses/08584187856776557038\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-9ltlvd/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-my6owol6p5vmm\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-02' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-02' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-02' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-02' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-02' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-02' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-02' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-02' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-02' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-02' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"6cf3b76e072dac1375c07a93b626cbe14066e595eaddfe6c237e5adb607cfb18","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"f8b319601237ac9c7268b0062159ff1f4b7f8c8d0188859aa2e726047d45afaa","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:449]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ccd460\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-skip-minor-72szdl-fpzdhp\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-skip-minor-72szdl-fpzdhp/operationStatuses/08584187856560051514\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-72szdl-nh8g5v/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-jmzrlgk5vtvcc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:449]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ccd460\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-skip-minor-72szdl-fpzdhp\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-skip-minor-72szdl-fpzdhp/operationStatuses/08584187856560051514\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-72szdl-nh8g5v/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-jmzrlgk5vtvcc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-15' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"7d85736bf241b9f867b72502bfc72e22e8cfdb38cb672219bf082896836de890","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"09d850fee8d4482ebf1662e590a2d7c18e5ee1a94fb6461738c273b8de3f2fec","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:92]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-21-dz2ll2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a5bea0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-21-dz2ll2-wb6r7g\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-dz2ll2-wb6r7g/operationStatuses/08584187855570807578\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-4bt6b9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-xgsfx5evhbp6k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:92]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-21-dz2ll2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a5bea0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-21-dz2ll2-wb6r7g\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-dz2ll2-wb6r7g/operationStatuses/08584187855570807578\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-4bt6b9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-xgsfx5evhbp6k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"80c003942a4162d1f86e430017d454720e2a035019794295dc9dd2164c0ffffe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"9c46978f0ed3a54b36dabe7a6501d7801ad35e5394d07bc559a3561381b97bc4","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:74]: failed to create customer resources with private key vault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0eac0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-private-kv-cluster-m8cdqs\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-private-kv-cluster-m8cdqs/operationStatuses/08584187857019650480\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4klntz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-had2isw6vkgre\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-09' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-09' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-09' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-09' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-09' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-09' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-09' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-09' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-09' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-09' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:74]: failed to create customer resources with private key vault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0eac0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-private-kv-cluster-m8cdqs\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-private-kv-cluster-m8cdqs/operationStatuses/08584187857019650480\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4klntz/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-had2isw6vkgre\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-09' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-09' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-09' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-09' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-09' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-09' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-09' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-09' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-09' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-09' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"82378d2a06f4d1cff77b6f242befb1da29bd775f27045972b08bb0689048a21b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"2a629b3f1740dd21ee93805f546fe87b423b07f78360ee922d1bf445ecf3dfa4","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:149]: failed to create customer resources for patch-tags cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001508da0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-patch-tags-cluster-rxqsg5\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-tags-cluster-rxqsg5/operationStatuses/08584187858042570035\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-s7b5tf/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fklo75htpxzxc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:149]: failed to create customer resources for patch-tags cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001508da0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-patch-tags-cluster-rxqsg5\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-tags-cluster-rxqsg5/operationStatuses/08584187858042570035\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-s7b5tf/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-fklo75htpxzxc\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-04' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"852b602c0f79564f03fa9d97e1fa7d72e613cea13404254b6057fb7bf06ea5a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"e8706800ce44d85c5e08639afcc158ec589751a94205e9c6f088906372091f15","test_name":"Customer should be able to update node pool labels and taints","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:81]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e9d140\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-np-labels-taints-xpnlw6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-np-labels-taints-xpnlw6/operationStatuses/08584187856711496599\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-8tcqz7/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-r6cuakilyx67k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:81]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e9d140\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-np-labels-taints-xpnlw6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-np-labels-taints-xpnlw6/operationStatuses/08584187856711496599\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-8tcqz7/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-r6cuakilyx67k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"85ff62400f638f5d183b1bdfc468cd60529dc5a32650acb0d1f38fff0a3445c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"9be64355d23290b23354c38f1b7f554f2fca88191b988cbd04b1aba006b5d893","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:92]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-22-tjhzrs\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d89ea0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-22-tjhzrs-sb4966\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-22-tjhzrs-sb4966/operationStatuses/08584187856891316107\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-jx9frs/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wku3kkbhewpoe\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:92]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-22-tjhzrs\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d89ea0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-22-tjhzrs-sb4966\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-22-tjhzrs-sb4966/operationStatuses/08584187856891316107\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-jx9frs/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-wku3kkbhewpoe\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-13' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"89a11b058c6aa04c320be8ba05aa2657ec28e6c44c54c698f8dd80e0792dee9c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"f736ef9440419869f9a42124315919fca1d54fb14827d32c7b38b76b128b63b3","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:122]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008c7f40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-gz5klc-gl25fs\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-gz5klc-gl25fs/operationStatuses/08584187856741962093\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-gz5klc-d5g8nh/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-hruihh5h6o7ho\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:122]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008c7f40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-gz5klc-gl25fs\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-gz5klc-gl25fs/operationStatuses/08584187856741962093\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-gz5klc-d5g8nh/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-hruihh5h6o7ho\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"95a989967a675f264e1f4bcced83a52739a298ca9a0872a98b8721efe58cf39f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"b046c6a32d513e33d83af7ab8fa2d1b47a2826112cfa149b753641439a2bbc7e","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:73]: failed to create customer resources for patch-name cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b0e360\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-patch-name-cluster-74x4gh\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-name-cluster-74x4gh/operationStatuses/08584187858034157061\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-85zpmt/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rr7tdjqoygobu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:73]: failed to create customer resources for patch-name cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b0e360\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-patch-name-cluster-74x4gh\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-patch-name-cluster-74x4gh/operationStatuses/08584187858034157061\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-85zpmt/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-rr7tdjqoygobu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-16' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"9b12d09ac5010d9777b649aaa7a50076c79d5e06cdd88c7509c5f259512b9d2f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"45090c2c419bd101ae14f34f3c7aff65a0e74f8b77db4f72780a3075712edc8c","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:88]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008af780\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-autoscale-cluster-bxqzwd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-autoscale-cluster-bxqzwd/operationStatuses/08584187856854920273\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-sv6z6b/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ssrwqrnhyyabs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:88]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008af780\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-autoscale-cluster-bxqzwd\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-autoscale-cluster-bxqzwd/operationStatuses/08584187856854920273\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-sv6z6b/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ssrwqrnhyyabs\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-05' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"b5d0d841d0e5a3832917bdce7c4d3876735990327161611f54fd8aca8fa08460","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"0ecbf6de3c2287d382dfb1382d72ecdfb7991fcd39ddd39d17003cb7b9c8867e","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:726]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b03a40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-dg-minor-rmcnxc-hmljs9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-dg-minor-rmcnxc-hmljs9/operationStatuses/08584187855656406602\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-rmcnxc-v65txx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3ubvi6leodlqa\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:726]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b03a40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-dg-minor-rmcnxc-hmljs9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-dg-minor-rmcnxc-hmljs9/operationStatuses/08584187855656406602\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-rmcnxc-v65txx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3ubvi6leodlqa\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"b6f89e87f02bc3e51456ac5ca0965809295678a3afc82c0f726af272fb718ea4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"5c14a3d5d7347f54fedb335f461389d65e240c6f552ed36843edf51b8f7aca73","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:88]: failed to create customer resources for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e118e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-autoscaling-hcp-cluster-htvsqw\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-autoscaling-hcp-cluster-htvsqw/operationStatuses/08584187858022867554\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-ws9p9j/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-e4ixwyxsfjnje\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:88]: failed to create customer resources for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e118e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-autoscaling-hcp-cluster-htvsqw\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-autoscaling-hcp-cluster-htvsqw/operationStatuses/08584187858022867554\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-ws9p9j/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-e4ixwyxsfjnje\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-19' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"b8c54d1421d857e8971a4ad6b9c30282dd4678e97372cb0a3be2db25d269a724","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"eeb1952c18d5c51217c6e4d3cd769ecad22c5d8a588a6573baefda069f410969","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009e7c20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-noedge-px4p2g-4l884k\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-noedge-px4p2g-4l884k/operationStatuses/08584187856731725299\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-px4p2g-q6krz9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-mnlsiq3q4kxek\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009e7c20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-noedge-px4p2g-4l884k\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-noedge-px4p2g-4l884k/operationStatuses/08584187856731725299\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-px4p2g-q6krz9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-mnlsiq3q4kxek\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-17' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c24344e40b2e6df48c3112a46c7ea23625876c760fa1c30e46fe33cb3ca55a0f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"3a44728b58e3784a0bc1b1190bcc142a94982b587dd2eb84d1600070724d9d1b","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:122]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100a6e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-4ltht9-7j6qsx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-4ltht9-7j6qsx/operationStatuses/08584187856773401940\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-4ltht9-5jkm6t/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-xb3lnijigxmoa\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:122]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100a6e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-4ltht9-7j6qsx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-4ltht9-7j6qsx/operationStatuses/08584187856773401940\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-4ltht9-5jkm6t/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-xb3lnijigxmoa\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-07' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c4bc9badd15d67dc204a3e89a31329d31ff8cbc835a95ff4757a7de1e407299e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"d254d85636e055f3359a15be80ed2fabb93ab6614622bc7f1d8cc270feaffcda","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:95]: failed to create customer resources for cluster \"cluster-candidate-5-0-9ldzcl\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e973a0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-5-0-9ldzcl-xl49gs\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-5-0-9ldzcl-xl49gs/operationStatuses/08584187856834194188\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-sfjl8m/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-io4laudmh4wje\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:95]: failed to create customer resources for cluster \"cluster-candidate-5-0-9ldzcl\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e973a0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-candidate-5-0-9ldzcl-xl49gs\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-candidate-5-0-9ldzcl-xl49gs/operationStatuses/08584187856834194188\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-sfjl8m/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-io4laudmh4wje\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c9549a6d44a0e6edfd382294934f1da91ae311cf120f45155dbc089825d8d591","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"ef6bd3bbf4b3e0af7eaf25a2f5fe9c5c68263ac76e5780532dea539d3d845569","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:70]: failed to create customer resources for cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f104e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-hcp-cluster-np-128-lp87gt\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-hcp-cluster-np-128-lp87gt/operationStatuses/08584187858042127885\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-mzr59f/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3ljyjjx2n7gxo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:70]: failed to create customer resources for cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f104e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-hcp-cluster-np-128-lp87gt\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-hcp-cluster-np-128-lp87gt/operationStatuses/08584187858042127885\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-mzr59f/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3ljyjjx2n7gxo\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-18' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"cc3e57eeaddf7b943ff69b16e424d01556a0ad460cba12fc5079b7275dc4fcfc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"23dae5e510cf6fc1472349e11722c9db724172e13f232de28160f84485722c12","test_name":"HCP Nodepools GPU instances creates and deletes a GPU nodepool in a single cluster","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:72]: failed to create customer resources for GPU nodepool cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a0c2c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-gpu-np-cp7dkx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-gpu-np-cp7dkx/operationStatuses/08584187856780553726\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-jqpcdg/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ruz4uxlifk3nm\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:72]: failed to create customer resources for GPU nodepool cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a0c2c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-gpu-np-cp7dkx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-gpu-np-cp7dkx/operationStatuses/08584187856780553726\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-gpu-nodepool-jqpcdg/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ruz4uxlifk3nm\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-06' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"dac55ed19994bef8baaf9b142a114d7a480295f8bc5db8ad114a48e9bbfd120d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"61e4114e885241be9f045ff405cf9eaf0292e0431e49cc17f288b4c872e63e4c","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:92]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-20-vh49qb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001281560\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-20-vh49qb-9mp8fg\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-vh49qb-9mp8fg/operationStatuses/08584187855617573436\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-5rdrhb/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ks6edxnijirju\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:92]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-20-vh49qb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001281560\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-20-vh49qb-9mp8fg\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-20-vh49qb-9mp8fg/operationStatuses/08584187855617573436\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-5rdrhb/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-ks6edxnijirju\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-10' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"db4fa3fa35f45a830cdb08e61f9e88cc4f9e0d99aa7c04e256c527824a29e4a8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"12563341c543c21bf1c5a4d7c3c5c4ea6b7793ce1d5980e5254a16a177239363","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:78]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000514c20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-disabled-image-registry-hcp-cluster-j5nhqc\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-disabled-image-registry-hcp-cluster-j5nhqc/operationStatuses/08584187856862566654\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-xm4sj4/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qmh74zogaodhg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:78]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000514c20\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-disabled-image-registry-hcp-cluster-j5nhqc\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-disabled-image-registry-hcp-cluster-j5nhqc/operationStatuses/08584187856862566654\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-xm4sj4/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-qmh74zogaodhg\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-01' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"e016c606d7d29e0081117fdf2179b789f8a1e00fd284fa6da19b53cba67500ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"d04a8bb5bf2682a7d190b66cff75f40dd541a772158cfcef2b9a073d056d9a63","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:276]: failed to create cluster customer resources for node-limit test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b1a640\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-node-limit-cluster-hxvbhg\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-hxvbhg/operationStatuses/08584187856876786135\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-7kzpgt/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:276]: failed to create cluster customer resources for node-limit test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b1a640\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-node-limit-cluster-hxvbhg\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-hxvbhg/operationStatuses/08584187856876786135\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-7kzpgt/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-08' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"e3b2fb62d4d5441f219ff92a8bf8f41fc6adea0eb600a99ea02fdb499b47534b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"2c40c0772c3444852c3ccece80cffa7412c6bfc314c90be2fd69f25803499fd3","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:85]: failed to create customer resources for TLS endpoint cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f84de0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-tls-endpoint-hcp-cluster-nkdnjx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-tls-endpoint-hcp-cluster-nkdnjx/operationStatuses/08584187858040367930\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-vbsggw/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oo7pyvaeowk44\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:85]: failed to create customer resources for TLS endpoint cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f84de0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-tls-endpoint-hcp-cluster-nkdnjx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-tls-endpoint-hcp-cluster-nkdnjx/operationStatuses/08584187858040367930\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-vbsggw/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-oo7pyvaeowk44\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-12' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"e6d9c4887f9df1ad8d1dac79a5306939a83adf585e2d4a0f53f214fcd365d52c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"9f2c914048875621b22570eae14cbf892862ceb56c80112cb4c588255b3e22f4","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:132]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000a623c0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-dcqfgd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: FailedIdentityOperation\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"FailedIdentityOperation\",\n \"message\": \"Identity operation for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-dcqfgd/providers/Microsoft.RedHatOpenShift/HCPOpenShiftClusters/delayed-rbac-cluster' failed with error 'Failed to perform resource identity operation. Status: 'BadRequest'. Response: '{\\\"error\\\":{\\\"code\\\":\\\"BadRequest\\\",\\\"message\\\":\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/aro-hcp-msi-container-int-shard0-00-13/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager' was not found.\\\"}}'.'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:132]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000a623c0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-dcqfgd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: FailedIdentityOperation\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"FailedIdentityOperation\",\n \"message\": \"Identity operation for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-dcqfgd/providers/Microsoft.RedHatOpenShift/HCPOpenShiftClusters/delayed-rbac-cluster' failed with error 'Failed to perform resource identity operation. Status: 'BadRequest'. Response: '{\\\"error\\\":{\\\"code\\\":\\\"BadRequest\\\",\\\"message\\\":\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/aro-hcp-msi-container-int-shard0-00-13/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager' was not found.\\\"}}'.'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"899b58022897cccfb9ea0c9de9b52c4b8e31f9ece177017a1c6420a0b8178411","failure_pattern":"ERROR CODE: FailedIdentityOperation; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: FailedIdentityOperation"}},{"row_id":"ee64938d7a8d391602cafc387069502e1b9aa917df4dd355c9c7b7d9337f7923","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"06adc16b81f24fca07589636090382f180e86c882cdb9101d32bd48780c5d2e9","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:81]: failed to create customer resources with private key vault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c04500\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cilium-cluster-bknd4f\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cilium-cluster-bknd4f/operationStatuses/08584187857020344075\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4gslc2/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-4vnk4izirdfzu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:81]: failed to create customer resources with private key vault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c04500\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cilium-cluster-bknd4f\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cilium-cluster-bknd4f/operationStatuses/08584187857020344075\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4gslc2/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-4vnk4izirdfzu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"f86a862723664bc154f080ccff869ac066b77b15a8e95169a5e86ff36e41c94e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","failed_at":"e2e","signature_id":"b6b3c5d38f6aa39e46c68bdba33f05c028c83dca68e64418e97d0b7bf7326b8c","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:87]: failed to create customer resources for admin credential lifecycle cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f9ec40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-admin-cred-lifecycle-pnnk7t-pb8dz9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-admin-cred-lifecycle-pnnk7t-pb8dz9/operationStatuses/08584187858025314704\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-9hbvc5/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-es7labf5uzmr2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:87]: failed to create customer resources for admin credential lifecycle cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f9ec40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-admin-cred-lifecycle-pnnk7t-pb8dz9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-admin-cred-lifecycle-pnnk7t-pb8dz9/operationStatuses/08584187858025314704\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-9hbvc5/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-es7labf5uzmr2\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-14' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":43,"failure_pattern_ids":["7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","899b58022897cccfb9ea0c9de9b52c4b8e31f9ece177017a1c6420a0b8178411"],"matched_occurrences":43,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071854534908448768","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-06-30T07:13:08Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071813581342511104","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-06-30T04:22:15Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}