{"meta":{"start_date":"2026-06-29","end_date":"2026-07-05","start_at":"2026-06-29T00:00:00Z","end_at":"2026-07-06T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T05:25:18Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":22,"failed_runs":7,"raw_occurrences":66,"matched_occurrences":66,"runs_affected":7},"rows":[{"environment":"int","failure_pattern_id":"7645ed77896a6fd87940286da3b1c170ec494c23e7d008ccd7b87a9b6b4511fa","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceGroupNotFound; detail message resource group '\u003cresource-group\u003e' could not be found.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":42,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.545454545454546,"anchor_occurrences":42,"after_last_push_count":42,"trend_counts":[0,0,42,0,0,0,0,0],"trend_range":"2026-06-28..2026-07-05 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes a GPU nodepool in a single cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:81]: failed to create customer resources with private key vault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c04500\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cilium-cluster-bknd4f\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cilium-cluster-bknd4f/operationStatuses/08584187857020344075\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4gslc2/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-4vnk4izirdfzu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:81]: failed to create customer resources with private key vault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c04500\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cilium-cluster-bknd4f\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cilium-cluster-bknd4f/operationStatuses/08584187857020344075\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4gslc2/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-4vnk4izirdfzu\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-03' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:92]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-21-dz2ll2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a5bea0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-21-dz2ll2-wb6r7g\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-dz2ll2-wb6r7g/operationStatuses/08584187855570807578\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-4bt6b9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-xgsfx5evhbp6k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:92]: failed to create customer resources for z-stream cluster \"cluster-zstream-4-21-dz2ll2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a5bea0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-21-dz2ll2-wb6r7g\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-21-dz2ll2-wb6r7g/operationStatuses/08584187855570807578\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-4bt6b9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-xgsfx5evhbp6k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-11' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:726]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b03a40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-dg-minor-rmcnxc-hmljs9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-dg-minor-rmcnxc-hmljs9/operationStatuses/08584187855656406602\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-rmcnxc-v65txx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3ubvi6leodlqa\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:726]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b03a40\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-dg-minor-rmcnxc-hmljs9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-dg-minor-rmcnxc-hmljs9/operationStatuses/08584187855656406602\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-rmcnxc-v65txx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-3ubvi6leodlqa\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"NotFound\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceGroupNotFound\\\",\\r\\n \\\"message\\\": \\\"Resource group 'aro-hcp-msi-container-int-shard0-00-00' could not be found.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"06adc16b81f24fca07589636090382f180e86c882cdb9101d32bd48780c5d2e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"09d850fee8d4482ebf1662e590a2d7c18e5ee1a94fb6461738c273b8de3f2fec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"0ecbf6de3c2287d382dfb1382d72ecdfb7991fcd39ddd39d17003cb7b9c8867e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"12563341c543c21bf1c5a4d7c3c5c4ea6b7793ce1d5980e5254a16a177239363","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"1b5e4b2b6c38660650be734f1ec5e7c9be92da2a62258a329a69870223072e3b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"23dae5e510cf6fc1472349e11722c9db724172e13f232de28160f84485722c12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"273823a019f7c9d93136bbebc8def0ef3e38de316e9a1602cbe8671a78eac958","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"2a629b3f1740dd21ee93805f546fe87b423b07f78360ee922d1bf445ecf3dfa4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"2c40c0772c3444852c3ccece80cffa7412c6bfc314c90be2fd69f25803499fd3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"34be6c410ebea8f42dd1fb2a969da344a6e92049f925aaf8aeb095d79883b63e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"3a44728b58e3784a0bc1b1190bcc142a94982b587dd2eb84d1600070724d9d1b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"45090c2c419bd101ae14f34f3c7aff65a0e74f8b77db4f72780a3075712edc8c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"500d13171fa535f3ee3716a480691bb574ee03c1626eec39d2336dd608b99b86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"5c14a3d5d7347f54fedb335f461389d65e240c6f552ed36843edf51b8f7aca73","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"61e4114e885241be9f045ff405cf9eaf0292e0431e49cc17f288b4c872e63e4c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"6f1e97cf86a258eea5ba652d10722e0ae8cb510308dde083aa0ecaf21bfc5680","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"7424c4f23072dd7f3e87770531a92600cee18a30634b64d9910f9b55929d3fe5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"77b40d648dced3d8e5d3fbbc5b5eb87c7f24b138a5b470586495c840e5a2f82d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"77f1b7824560703925a292a7041cdf47ad3a12c83a025c97a7c65a17fa472039","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"98d5703ec40c112dd994dbb997c832fbbcca1673f291db6c3f5045ebbfcbd9d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"9be64355d23290b23354c38f1b7f554f2fca88191b988cbd04b1aba006b5d893","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"9c46978f0ed3a54b36dabe7a6501d7801ad35e5394d07bc559a3561381b97bc4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"9ccc60807bb3d0f8b59a971d84e36cb511ac554bfb0ecafd693050914f979d3e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"a35a4da4a7237e28b701f0b99f0eecdd000ba02a38c029b5c2d2fa6da870fdd6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"a57e91da831d03a19b0c0cc92579b0175463252c0a61fb07ec0c35f9996cb865","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"a83201a94a704c00ee7627b06a1da25132db69b4968b624673a7b64ba3520b99","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"b046c6a32d513e33d83af7ab8fa2d1b47a2826112cfa149b753641439a2bbc7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"b6b3c5d38f6aa39e46c68bdba33f05c028c83dca68e64418e97d0b7bf7326b8c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"bcc21619c009b761b74efa81cefb7f2dae7ecf0fb9430884ab2d08b2f5359da7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"cae70c071af5d80d4dd2b47aff48a145a85ba023612468bcb95c49586bec0bda","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"d04a8bb5bf2682a7d190b66cff75f40dd541a772158cfcef2b9a073d056d9a63","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"d254d85636e055f3359a15be80ed2fabb93ab6614622bc7f1d8cc270feaffcda","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"d46662b6c5c3b5077517422943c458d95651ea56cb570be219e9b65d5e7778f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"e8706800ce44d85c5e08639afcc158ec589751a94205e9c6f088906372091f15","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"ea6a6cd90ad562eb3c02d92f54fb70dbc35b9e003f1ad7afcf96d803187301b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"eeb1952c18d5c51217c6e4d3cd769ecad22c5d8a588a6573baefda069f410969","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"ef6bd3bbf4b3e0af7eaf25a2f5fe9c5c68263ac76e5780532dea539d3d845569","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"f1cc88e2e1050bbe52cb39e7358522c6a4266e2a4f3b789c3d67dfbd74510560","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"f2f35e2f8b26e8d5fbf2b68c98c1d4ffa92106a6be0d5cc595477381dee1c8d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"f736ef9440419869f9a42124315919fca1d54fb14827d32c7b38b76b128b63b3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"f8b319601237ac9c7268b0062159ff1f4b7f8c8d0188859aa2e726047d45afaa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"f9b971e49b61dfee9197525ff123c2091e020c73e8f671a4243dac9f8077c16b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-j5525s, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-j5525s in resource group rg-candidate-4-21-k8l4nl, error: failed waiting for cluster=\"cluster-candidate-4-21-j5525s\" in resourcegroup=\"rg-candidate-4-21-k8l4nl\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-j5525s in resource group rg-candidate-4-21-k8l4nl, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":18,"runs_affected":3,"failed_runs":3,"run_impact_percent":13.636363636363637,"anchor_occurrences":18,"after_last_push_count":18,"trend_counts":[0,0,0,1,1,16,0,0],"trend_range":"2026-06-28..2026-07-05 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-01","2026-06-08","2026-06-15","2026-06-22"],"prior_runs_affected":41,"prior_last_seen_at":"2026-06-26T21:07:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000970840\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-pbl25v\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group complex-cilium-kv-pbl25v, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000970840\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-pbl25v\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group complex-cilium-kv-pbl25v, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0016f4720\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-b8zjkr\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-b8zjkr, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0016f4720\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-b8zjkr\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-b8zjkr, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000ac29f0\u003e: \n failed to create HCP cluster oidc-wi-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-x468p9, error: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-x468p9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-x468p9, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000ac29f0\u003e: \n failed to create HCP cluster oidc-wi-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-x468p9, error: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-x468p9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-x468p9, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072975803280265216","occurred_at":"2026-07-03T09:28:40Z","signature_id":"2569c026db6b8821bdeb31c421f46d68fc25abe7177dccf53d1aa63617757aa9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072975803280265216","occurred_at":"2026-07-03T09:28:40Z","signature_id":"27b1479a0ac9b5982274541ba59185c810e93ddb29121fa3a531e5bfed2ca11c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072975803280265216","occurred_at":"2026-07-03T09:28:40Z","signature_id":"30076112bf613723bf534f53ed5b5778bd7d33eca5a8cb1cdad77e1c9d60d2fb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072975803280265216","occurred_at":"2026-07-03T09:28:40Z","signature_id":"3a684e506c3d6898dedf3025884a132cb3fdfeb16b6fece6dd87ff6766399508","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072975803280265216","occurred_at":"2026-07-03T09:28:40Z","signature_id":"4265cb4f3d10c246e7b38ae05bae1bbce3e6025f1f0d8b538776e846d115636e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072975803280265216","occurred_at":"2026-07-03T09:28:40Z","signature_id":"48d4b8497840d1107797350b5b152e5faab80c408cc5d10e94004453121fdcda","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072975803280265216","occurred_at":"2026-07-03T09:28:40Z","signature_id":"49face5d90a38a1267ea2702c8e58490c582cf880ab2a5547c68a6964be246fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072975803280265216","occurred_at":"2026-07-03T09:28:40Z","signature_id":"4f556dcf3ae24e9f147889e9df53a923bd3f7d5e1c3af42150095c647477b84f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072975803280265216","occurred_at":"2026-07-03T09:28:40Z","signature_id":"6a5e2028c4e9bce8f95fe59e56ecec82568fa732ed6684d1ce677bc82b677f95","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072975803280265216","occurred_at":"2026-07-03T09:28:40Z","signature_id":"732a2e2ccc0a605f891283605e1f34a7e235d0de7a40a64a4387bebcf0d73b8e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072975803280265216","occurred_at":"2026-07-03T09:28:40Z","signature_id":"79fbf5a88689a013ed6ef15aaa8ee370a92acac479cf7354e392df2d6b657aef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072975803280265216","occurred_at":"2026-07-03T09:28:40Z","signature_id":"7a7426c4a153dbd23894db4ab6d0420852ba63db05161c08d89596bb29f63dc2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072975803280265216","occurred_at":"2026-07-03T09:28:40Z","signature_id":"add1036b94563f224870631213884d53c74292660cf6c7388db6cfdbf78e4dcb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072975803280265216","occurred_at":"2026-07-03T09:28:40Z","signature_id":"afa707f875a96beab4e8658eb48b5aa01bc214e5bab87968f2ffd5b136f1bd28","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072975803280265216","occurred_at":"2026-07-03T09:28:40Z","signature_id":"efa44a9e54542ed041dd7b8393ecd0bb4e2bff3a522d6f31fc27a15d3567183c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072975803280265216","occurred_at":"2026-07-03T09:28:40Z","signature_id":"f40da8e3949ba7d3c8baf1bcd7aeec40a29cd77ac04f2fba29e29ad137945430","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072691050320760832","occurred_at":"2026-07-02T14:37:09Z","signature_id":"62d041a5400a290af552026b87c1b4cb3694b15c519246431f59527e6713fa26","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072159025578905600","occurred_at":"2026-07-01T03:23:05Z","signature_id":"b7728add38df7c8d3e71da0ca69b322ad952614ceb98b114cad233e9133ae837","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"52d9258f03a60774b49eaa70a514574e25b40d0eeb6ddfa13f252e24164f6be2","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-gtq44x\" to finish deleting: context canceled","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes a GPU nodepool in a single cluster","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.545454545454546,"anchor_occurrences":2,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,2,0,0],"trend_range":"2026-06-28..2026-07-05 UTC","prior_windows_present":2,"prior_window_starts":["2026-06-01","2026-06-08"],"prior_runs_affected":4,"prior_last_seen_at":"2026-06-11T00:26:01Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes a GPU nodepool in a single cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0013f2330\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-5v4tq6\" in resourcegroup=\"rg-zstream-upgrade-4-22-wl4tn4\" to finish deleting: context canceled\n ...\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0013f2330\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-5v4tq6\" in resourcegroup=\"rg-zstream-upgrade-4-22-wl4tn4\" to finish deleting: context canceled\n ...\noccurred \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016ca1b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-gtq44x\" to finish deleting: context canceled\n ...\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0016ca1b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-gtq44x\" to finish deleting: context canceled\n ...\noccurred \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072975803280265216","occurred_at":"2026-07-03T09:28:40Z","signature_id":"0b0b29ab1e5c93d6c87eb39041439bb16d522d169f78a4acdf629c4761a95b94","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072975803280265216","occurred_at":"2026-07-03T09:28:40Z","signature_id":"1b9248af657fe52f0af7a42c5943ed7770fd661595dd49f9aa4ac49de9acf25e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"a880b7dbbe8686372b606e0212fd3742bc75c3c0a9c545bae9481b243ed0f55b","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-2jm9fc in resource group rg-candidate-4-20-tjx795, error: failed waiting for nodepool=\"np-2jm9fc\" for cluster \"cluster-candidate-4-20-2jm9fc\" in resourcegroup=\"rg-candidate-4-20-tjx795\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.545454545454546,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-06-28..2026-07-05 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-01","2026-06-08","2026-06-15","2026-06-22"],"prior_runs_affected":21,"prior_last_seen_at":"2026-06-26T01:04:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-2jm9fc\" for cluster \"cluster-candidate-4-20-2jm9fc\"\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0014d4ff0\u003e: \n failed to create NodePool np-2jm9fc, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-2jm9fc in resource group rg-candidate-4-20-tjx795, error: failed waiting for nodepool=\"np-2jm9fc\" for cluster \"cluster-candidate-4-20-2jm9fc\" in resourcegroup=\"rg-candidate-4-20-tjx795\" to finish creating: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-2jm9fc\" for cluster \"cluster-candidate-4-20-2jm9fc\"\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0014d4ff0\u003e: \n failed to create NodePool np-2jm9fc, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-2jm9fc in resource group rg-candidate-4-20-tjx795, error: failed waiting for nodepool=\"np-2jm9fc\" for cluster \"cluster-candidate-4-20-2jm9fc\" in resourcegroup=\"rg-candidate-4-20-tjx795\" to finish creating: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072845222861082624","occurred_at":"2026-07-03T00:49:32Z","signature_id":"abb710fa3f860f16d80971d5f1840b91873c635c62d25f4aff2e644736a9ec5f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"899b58022897cccfb9ea0c9de9b52c4b8e31f9ece177017a1c6420a0b8178411","failure_pattern":"ERROR CODE: FailedIdentityOperation; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: FailedIdentityOperation","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.545454545454546,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-06-28..2026-07-05 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:132]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000a623c0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-dcqfgd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: FailedIdentityOperation\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"FailedIdentityOperation\",\n \"message\": \"Identity operation for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-dcqfgd/providers/Microsoft.RedHatOpenShift/HCPOpenShiftClusters/delayed-rbac-cluster' failed with error 'Failed to perform resource identity operation. Status: 'BadRequest'. Response: '{\\\"error\\\":{\\\"code\\\":\\\"BadRequest\\\",\\\"message\\\":\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/aro-hcp-msi-container-int-shard0-00-13/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager' was not found.\\\"}}'.'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:132]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000a623c0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-dcqfgd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: FailedIdentityOperation\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"FailedIdentityOperation\",\n \"message\": \"Identity operation for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-dcqfgd/providers/Microsoft.RedHatOpenShift/HCPOpenShiftClusters/delayed-rbac-cluster' failed with error 'Failed to perform resource identity operation. Status: 'BadRequest'. Response: '{\\\"error\\\":{\\\"code\\\":\\\"BadRequest\\\",\\\"message\\\":\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/aro-hcp-msi-container-int-shard0-00-13/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager' was not found.\\\"}}'.'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2071922878621683712","occurred_at":"2026-06-30T11:44:43Z","signature_id":"9f2c914048875621b22570eae14cbf892862ceb56c80112cb4c588255b3e22f4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"e457d9ded8188e2e5d3bf94632039456ab0d3f40aab17c66a163827314f30476","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.545454545454546,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-06-28..2026-07-05 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-08"],"prior_runs_affected":2,"prior_last_seen_at":"2026-06-11T05:24:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc00230e510\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-l2r9rs, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-l2r9rs\" to finish creating: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc00230e510\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-l2r9rs, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-l2r9rs\" to finish creating: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072353168804548608","occurred_at":"2026-07-01T16:14:32Z","signature_id":"53cde8c061a06de6fd3c0817e7e6bbd870277d43e54f452cf7f9f131f870c6a1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":4.545454545454546,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-06-28..2026-07-05 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-15"],"prior_runs_affected":2,"prior_last_seen_at":"2026-06-19T00:47:56Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-znmjdl\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000beaa80\u003e: \n route was never reachable: Get \"https://agnhost-e2e-serving-app-c4b8k.apps.aro.w5t4t0w2o2r4p4u.n9tm.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-znmjdl\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000beaa80\u003e: \n route was never reachable: Get \"https://agnhost-e2e-serving-app-c4b8k.apps.aro.w5t4t0w2o2r4p4u.n9tm.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2072253121135906816","occurred_at":"2026-07-01T09:36:59Z","signature_id":"f371c2b60e4490e865b62413ca38c4eaf15738649c0b8390382cebf53dd5ceb6","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}