{"meta":{"start_date":"2026-06-30","end_date":"2026-06-30","start_at":"2026-06-30T00:00:00Z","end_at":"2026-07-01T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T08:26:29Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":5,"failed_runs":1,"raw_occurrences":43,"matched_occurrences":43,"runs_affected":1},"rows":[{"environment":"int","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":42,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":42,"after_last_push_count":42,"trend_counts":[0,0,0,0,0,0,42],"trend_range":"2026-06-24..2026-06-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes a GPU nodepool in a single cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:81]: failed to create customer resources with private key vault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c04500\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cilium-cluster-bknd4f\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cilium-cluster-bknd4f/operationStatuses/08584187857020344075\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4gslc2/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-4vnk4izirdfzu\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"NotFound\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n    \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:81]: failed to create customer resources with private key vault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c04500\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cilium-cluster-bknd4f\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cilium-cluster-bknd4f/operationStatuses/08584187857020344075\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4gslc2/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-4vnk4izirdfzu\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"NotFound\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n    \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:92]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-21-dz2ll2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a5bea0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-21-dz2ll2-wb6r7g\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-dz2ll2-wb6r7g/operationStatuses/08584187855570807578\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-4bt6b9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-xgsfx5evhbp6k\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"NotFound\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n    \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:92]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-21-dz2ll2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a5bea0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-21-dz2ll2-wb6r7g\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-dz2ll2-wb6r7g/operationStatuses/08584187855570807578\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-4bt6b9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-xgsfx5evhbp6k\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"NotFound\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n    \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:726]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b03a40\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-dg-minor-rmcnxc-hmljs9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-dg-minor-rmcnxc-hmljs9/operationStatuses/08584187855656406602\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-rmcnxc-v65txx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3ubvi6leodlqa\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"NotFound\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n    \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:726]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b03a40\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-dg-minor-rmcnxc-hmljs9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-dg-minor-rmcnxc-hmljs9/operationStatuses/08584187855656406602\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-rmcnxc-v65txx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3ubvi6leodlqa\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"NotFound\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n    \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"06adc16b81f24fca07589636090382f180e86c882cdb9101d32bd48780c5d2e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"09d850fee8d4482ebf1662e590a2d7c18e5ee1a94fb6461738c273b8de3f2fec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"0ecbf6de3c2287d382dfb1382d72ecdfb7991fcd39ddd39d17003cb7b9c8867e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"12563341c543c21bf1c5a4d7c3c5c4ea6b7793ce1d5980e5254a16a177239363","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"1b5e4b2b6c38660650be734f1ec5e7c9be92da2a62258a329a69870223072e3b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"23dae5e510cf6fc1472349e11722c9db724172e13f232de28160f84485722c12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"273823a019f7c9d93136bbebc8def0ef3e38de316e9a1602cbe8671a78eac958","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"2a629b3f1740dd21ee93805f546fe87b423b07f78360ee922d1bf445ecf3dfa4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"2c40c0772c3444852c3ccece80cffa7412c6bfc314c90be2fd69f25803499fd3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"34be6c410ebea8f42dd1fb2a969da344a6e92049f925aaf8aeb095d79883b63e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"3a44728b58e3784a0bc1b1190bcc142a94982b587dd2eb84d1600070724d9d1b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"45090c2c419bd101ae14f34f3c7aff65a0e74f8b77db4f72780a3075712edc8c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"500d13171fa535f3ee3716a480691bb574ee03c1626eec39d2336dd608b99b86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"5c14a3d5d7347f54fedb335f461389d65e240c6f552ed36843edf51b8f7aca73","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"61e4114e885241be9f045ff405cf9eaf0292e0431e49cc17f288b4c872e63e4c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"6f1e97cf86a258eea5ba652d10722e0ae8cb510308dde083aa0ecaf21bfc5680","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"7424c4f23072dd7f3e87770531a92600cee18a30634b64d9910f9b55929d3fe5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"77b40d648dced3d8e5d3fbbc5b5eb87c7f24b138a5b470586495c840e5a2f82d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"77f1b7824560703925a292a7041cdf47ad3a12c83a025c97a7c65a17fa472039","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"98d5703ec40c112dd994dbb997c832fbbcca1673f291db6c3f5045ebbfcbd9d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"9be64355d23290b23354c38f1b7f554f2fca88191b988cbd04b1aba006b5d893","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"9c46978f0ed3a54b36dabe7a6501d7801ad35e5394d07bc559a3561381b97bc4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"9ccc60807bb3d0f8b59a971d84e36cb511ac554bfb0ecafd693050914f979d3e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"a35a4da4a7237e28b701f0b99f0eecdd000ba02a38c029b5c2d2fa6da870fdd6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"a57e91da831d03a19b0c0cc92579b0175463252c0a61fb07ec0c35f9996cb865","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"a83201a94a704c00ee7627b06a1da25132db69b4968b624673a7b64ba3520b99","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"b046c6a32d513e33d83af7ab8fa2d1b47a2826112cfa149b753641439a2bbc7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"b6b3c5d38f6aa39e46c68bdba33f05c028c83dca68e64418e97d0b7bf7326b8c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"bcc21619c009b761b74efa81cefb7f2dae7ecf0fb9430884ab2d08b2f5359da7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"cae70c071af5d80d4dd2b47aff48a145a85ba023612468bcb95c49586bec0bda","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"d04a8bb5bf2682a7d190b66cff75f40dd541a772158cfcef2b9a073d056d9a63","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"d254d85636e055f3359a15be80ed2fabb93ab6614622bc7f1d8cc270feaffcda","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"d46662b6c5c3b5077517422943c458d95651ea56cb570be219e9b65d5e7778f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"e8706800ce44d85c5e08639afcc158ec589751a94205e9c6f088906372091f15","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"ea6a6cd90ad562eb3c02d92f54fb70dbc35b9e003f1ad7afcf96d803187301b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"eeb1952c18d5c51217c6e4d3cd769ecad22c5d8a588a6573baefda069f410969","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"ef6bd3bbf4b3e0af7eaf25a2f5fe9c5c68263ac76e5780532dea539d3d845569","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"f1cc88e2e1050bbe52cb39e7358522c6a4266e2a4f3b789c3d67dfbd74510560","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"f2f35e2f8b26e8d5fbf2b68c98c1d4ffa92106a6be0d5cc595477381dee1c8d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"f736ef9440419869f9a42124315919fca1d54fb14827d32c7b38b76b128b63b3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"f8b319601237ac9c7268b0062159ff1f4b7f8c8d0188859aa2e726047d45afaa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"f9b971e49b61dfee9197525ff123c2091e020c73e8f671a4243dac9f8077c16b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"899b58022897cccfb9ea0c9de9b52c4b8e31f9ece177017a1c6420a0b8178411","failure_pattern":"ERROR CODE: FailedIdentityOperation; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: FailedIdentityOperation","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-24..2026-06-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:132]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a623c0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-dcqfgd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: FailedIdentityOperation\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"FailedIdentityOperation\",\n        \"message\": \"Identity operation for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-dcqfgd/providers/Microsoft.RedHatOpenShift/HCPOpenShiftClusters/delayed-rbac-cluster' failed with error 'Failed to perform resource identity operation. Status: 'BadRequest'. Response: '{\\\"error\\\":{\\\"code\\\":\\\"BadRequest\\\",\\\"message\\\":\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/aro-hcp-msi-container-int-shard0-00-13/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager' was not found.\\\"}}'.'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:132]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a623c0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-dcqfgd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: FailedIdentityOperation\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"FailedIdentityOperation\",\n        \"message\": \"Identity operation for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-dcqfgd/providers/Microsoft.RedHatOpenShift/HCPOpenShiftClusters/delayed-rbac-cluster' failed with error 'Failed to perform resource identity operation. Status: 'BadRequest'. Response: '{\\\"error\\\":{\\\"code\\\":\\\"BadRequest\\\",\\\"message\\\":\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/aro-hcp-msi-container-int-shard0-00-13/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager' was not found.\\\"}}'.'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"9f2c914048875621b22570eae14cbf892862ceb56c80112cb4c588255b3e22f4","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}