{"meta":{"start_date":"2026-07-20","end_date":"2026-07-26","start_at":"2026-07-20T00:00:00Z","end_at":"2026-07-27T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T03:06:00Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":29,"failed_runs":10,"raw_occurrences":10,"matched_occurrences":10,"runs_affected":10},"rows":[{"environment":"int","failure_pattern_id":"f90cae3228fe72b2427515e7c51834eebdc3e73c0b1aa6bca2abedd53a2e7578","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"arm64-vm-cluster-nwkxqmmqnhz4\", caused by: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-nwkxqmmqnhz4\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-nwkxqmmqnhz4, error: context deadline exceeded, error: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-nwkxqmmqnhz4\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-nwkxqmmqnhz4, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.896551724137931,"anchor_occurrences":2,"after_last_push_count":7,"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-06","2026-07-13"],"prior_runs_affected":6,"prior_last_seen_at":"2026-07-14T09:05:58Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc00139e2a0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"cni-cilium-nwkfkzbgddrj\", caused by: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-nwkfkzbgddrj\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cilium-cl in resource group cni-cilium-nwkfkzbgddrj, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-nwkfkzbgddrj\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cilium-cl in resource group cni-cilium-nwkfkzbgddrj, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc00139e2a0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"cni-cilium-nwkfkzbgddrj\", caused by: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-nwkfkzbgddrj\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cilium-cl in resource group cni-cilium-nwkfkzbgddrj, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-nwkfkzbgddrj\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cilium-cl in resource group cni-cilium-nwkfkzbgddrj, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc0013126d8\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"arm64-vm-cluster-nwkxqmmqnhz4\", caused by: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-nwkxqmmqnhz4\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-nwkxqmmqnhz4, error: context deadline exceeded, error: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-nwkxqmmqnhz4\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-nwkxqmmqnhz4, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc0013126d8\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"arm64-vm-cluster-nwkxqmmqnhz4\", caused by: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-nwkxqmmqnhz4\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-nwkxqmmqnhz4, error: context deadline exceeded, error: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-nwkxqmmqnhz4\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-nwkxqmmqnhz4, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2079974384990162944","occurred_at":"2026-07-22T16:58:32Z","signature_id":"fd02116e1ccb31bdbec9089291a8633722038583ad170e6616b532db92aedd0d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2079794933505462272","occurred_at":"2026-07-22T05:05:27Z","signature_id":"5f8cab9d8b29c0f99d2e01b7a1d33cd9a44feaddba2fb47085a86a20b6ba0d58","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"e0eca219c3110341e80c5a3f64ae984723abc1e1208ac11199b51c76f20de3e7","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.896551724137931,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,1,0,0,1,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-13"],"prior_runs_affected":3,"prior_last_seen_at":"2026-07-17T23:03:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:219]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00133f200\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5lxcfm9xd9kt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5ed2bba9-602d-44bd-909e-acec409b25d1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5ed2bba9-602d-44bd-909e-acec409b25d1\",\n      \"name\": \"5ed2bba9-602d-44bd-909e-acec409b25d1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T02:44:24.422048854Z\",\n      \"endTime\": \"2026-07-23T03:03:26.723870761Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:219]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00133f200\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5lxcfm9xd9kt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5ed2bba9-602d-44bd-909e-acec409b25d1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5ed2bba9-602d-44bd-909e-acec409b25d1\",\n      \"name\": \"5ed2bba9-602d-44bd-909e-acec409b25d1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-23T02:44:24.422048854Z\",\n      \"endTime\": \"2026-07-23T03:03:26.723870761Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001b16d68\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-bznnwq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f6c6dfd-730b-4056-b31f-c855a36148c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f6c6dfd-730b-4056-b31f-c855a36148c6\",\n      \"name\": \"5f6c6dfd-730b-4056-b31f-c855a36148c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T12:10:01.730836206Z\",\n      \"endTime\": \"2026-07-20T12:29:12.036855578Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001b16d68\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-bznnwq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f6c6dfd-730b-4056-b31f-c855a36148c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f6c6dfd-730b-4056-b31f-c855a36148c6\",\n      \"name\": \"5f6c6dfd-730b-4056-b31f-c855a36148c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T12:10:01.730836206Z\",\n      \"endTime\": \"2026-07-20T12:29:12.036855578Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2080106292172558336","occurred_at":"2026-07-23T01:42:33Z","signature_id":"a349e42ef865c798944b8abf4db6a1fd5623f19f19ebcae9bbc4e0d8f4024ac0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2079169422496370688","occurred_at":"2026-07-20T11:39:54Z","signature_id":"dc8db9fd86a3c83f8223654a97a644e971bade6eee745eada7761d3cd8154ebb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"73fe562a09e2fbf4240bf4a6cdc7b529ab336a8a5783177a17916fe1bc09aba8","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentActive; detail message The deployment with resource id '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '7/21/2026 11:31:54 PM' with correlationId '\u003cuuid\u003e', and will expire at '7/28/2026 11:31:54 PM' if it does not complete before then.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:81]: failed to create customer resources for first cluster \"basic-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000efef20\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster-lwhsj4\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-lwhsj4/operationStatuses/08584169301754153997\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"DeploymentActive\\\",\\r\\n    \\\"message\\\": \\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-fn5csp4xjr4l/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eqbgltuyzhpg4' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '7/21/2026 11:31:54 PM' with correlationId '84edb00d-eb04-47f9-bfb5-f5789781106f', and will expire at '7/28/2026 11:31:54 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\"\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"DeploymentActive\\\",\\r\\n    \\\"message\\\": \\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-int-shard0-00-14/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '7/21/2026 11:31:54 PM' with correlationId '84edb00d-eb04-47f9-bfb5-f5789781106f', and will expire at '7/28/2026 11:31:54 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:81]: failed to create customer resources for first cluster \"basic-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000efef20\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster-lwhsj4\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-lwhsj4/operationStatuses/08584169301754153997\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"DeploymentActive\\\",\\r\\n    \\\"message\\\": \\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-fn5csp4xjr4l/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eqbgltuyzhpg4' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '7/21/2026 11:31:54 PM' with correlationId '84edb00d-eb04-47f9-bfb5-f5789781106f', and will expire at '7/28/2026 11:31:54 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\"\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"DeploymentActive\\\",\\r\\n    \\\"message\\\": \\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-int-shard0-00-14/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '7/21/2026 11:31:54 PM' with correlationId '84edb00d-eb04-47f9-bfb5-f5789781106f', and will expire at '7/28/2026 11:31:54 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2079707293942812672","occurred_at":"2026-07-21T23:17:12Z","signature_id":"768dbaddad1e3a75bcf5788abfc19fa8ff2bb66236d6627d69f931b237e0c92b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"90dc413b9b32410c56e01198bcd6508fef157c78b13b631dfaa98ec6f0de2c54","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:79]: failed to create customer resources for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c2a6a0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cx-rg-hcp-cluster-r59cl2\" in resourcegroup=\"cx-rg-cluster-7z2n9v\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/cx-rg-cluster-7z2n9v/providers/Microsoft.Resources/deployments/customer-infra-cx-rg-hcp-cluster-r59cl2/operationStatuses/08584170421377078879\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n    \\\"message\\\": \\\"The vault name 'cust-kv-hqzib2x2rf23w' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:79]: failed to create customer resources for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c2a6a0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cx-rg-hcp-cluster-r59cl2\" in resourcegroup=\"cx-rg-cluster-7z2n9v\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/cx-rg-cluster-7z2n9v/providers/Microsoft.Resources/deployments/customer-infra-cx-rg-hcp-cluster-r59cl2/operationStatuses/08584170421377078879\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n    \\\"message\\\": \\\"The vault name 'cust-kv-hqzib2x2rf23w' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2079233040621506560","occurred_at":"2026-07-20T15:52:41Z","signature_id":"f56e1218a6562e503425c136b7134c08177a716a45edab4a6fb6025f5d756e70","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"a85096fd855e05a29939c294c71008436f6e47e4b0bc7a019252cf05c40d05fb","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-pthb4x/cluster-candidate-4-21-tb29zs should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d2b5e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-tb29zs: failed waiting for cluster=\"cluster-candidate-4-21-tb29zs\" in resourcegroup=\"rg-candidate-4-21-pthb4x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/964e61e7-c52a-4011-b322-68cdf175e817\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/964e61e7-c52a-4011-b322-68cdf175e817\",\n      \"name\": \"964e61e7-c52a-4011-b322-68cdf175e817\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T06:11:14.467304676Z\",\n      \"endTime\": \"2026-07-20T06:30:23.960698535Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-pthb4x/cluster-candidate-4-21-tb29zs should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d2b5e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-tb29zs: failed waiting for cluster=\"cluster-candidate-4-21-tb29zs\" in resourcegroup=\"rg-candidate-4-21-pthb4x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/964e61e7-c52a-4011-b322-68cdf175e817\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/964e61e7-c52a-4011-b322-68cdf175e817\",\n      \"name\": \"964e61e7-c52a-4011-b322-68cdf175e817\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T06:11:14.467304676Z\",\n      \"endTime\": \"2026-07-20T06:30:23.960698535Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2079080505143201792","occurred_at":"2026-07-20T05:46:34Z","signature_id":"761decc6fed7e79c0cc9e0d675d6f914544018b9bc09c92a11928f6b40080b83","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"cf6ca3be66fe9781f7098aaa5a5aadda4cfd59f4b600a60b3f48ef63f5158b75","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-9cldvc\" resourcegroup=\"rg-np-version-upgrade-9cldvc-x78hlk\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-9cldvc in resource group rg-np-version-upgrade-9cldvc-x78hlk, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:210]: failed to upgrade node pool npupgrade-4-20 to version 4.21.25\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0027f6060\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-9cldvc\" resourcegroup=\"rg-np-version-upgrade-9cldvc-x78hlk\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-9cldvc in resource group rg-np-version-upgrade-9cldvc-x78hlk, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:210]: failed to upgrade node pool npupgrade-4-20 to version 4.21.25\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0027f6060\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-9cldvc\" resourcegroup=\"rg-np-version-upgrade-9cldvc-x78hlk\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-9cldvc in resource group rg-np-version-upgrade-9cldvc-x78hlk, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2079302291038408704","occurred_at":"2026-07-20T20:27:52Z","signature_id":"b761cc7fc7a274620a3c70812af91772e6e033e7f8c7be195797e6792badcc38","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-s2bxkgdk9qhh\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-s2bxkgdk9qhh, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-22","2026-06-29","2026-07-06","2026-07-13"],"prior_runs_affected":21,"prior_last_seen_at":"2026-07-14T04:26:45Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00178a000\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-s2bxkgdk9qhh\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-s2bxkgdk9qhh, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00178a000\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-s2bxkgdk9qhh\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-s2bxkgdk9qhh, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2080767014686691328","occurred_at":"2026-07-24T21:28:09Z","signature_id":"c71dfd3691a0bf264e855f8edbee5600d2ab5db267f345772ccdd0af551ee363","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-07-19..2026-07-26 UTC","prior_windows_present":3,"prior_window_starts":["2026-06-29","2026-07-06","2026-07-13"],"prior_runs_affected":5,"prior_last_seen_at":"2026-07-14T20:05:58Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-r7d6d5\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a13320\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-sample-app-sqn8d.apps.aro.r8k3t7v7s5w7d4g.o85n.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-r7d6d5\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a13320\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-sample-app-sqn8d.apps.aro.r8k3t7v7s5w7d4g.o85n.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2079517159439994880","occurred_at":"2026-07-21T10:41:40Z","signature_id":"0f3156c77142aacb39abfb6839227b772d24678afa92ad8cb29b9267a2ec6063","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}