{"meta":{"start_date":"2026-07-27","end_date":"2026-08-02","start_at":"2026-07-27T00:00:00Z","end_at":"2026-08-03T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T02:26:32Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":42,"failed_runs":22,"raw_occurrences":76,"matched_occurrences":76,"runs_affected":22},"rows":[{"environment":"int","failure_pattern_id":"ac227089170e13ef6a47a3b4633ad2059cb4ac90e0b8628a57dcc930f8b4a4a2","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":40,"runs_affected":2,"failed_runs":2,"run_impact_percent":4.761904761904762,"anchor_occurrences":40,"after_last_push_count":40,"trend_counts":[0,1,0,0,0,39,0,0],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes a GPU nodepool in a single cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:105]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d5ecc0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rdjmcbb8c6qk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffeea5b7-a982-4432-9516-cc5e00f40479\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffeea5b7-a982-4432-9516-cc5e00f40479\",\n      \"name\": \"ffeea5b7-a982-4432-9516-cc5e00f40479\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-31T09:13:21.697734556Z\",\n      \"endTime\": \"2026-07-31T09:32:37.56053701Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:105]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d5ecc0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rdjmcbb8c6qk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffeea5b7-a982-4432-9516-cc5e00f40479\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffeea5b7-a982-4432-9516-cc5e00f40479\",\n      \"name\": \"ffeea5b7-a982-4432-9516-cc5e00f40479\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-31T09:13:21.697734556Z\",\n      \"endTime\": \"2026-07-31T09:32:37.56053701Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010db320\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-mclz4shl48hz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5247c381-2bd6-4d36-bde2-1113626087c1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5247c381-2bd6-4d36-bde2-1113626087c1\",\n      \"name\": \"5247c381-2bd6-4d36-bde2-1113626087c1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-31T09:58:02.878065641Z\",\n      \"endTime\": \"2026-07-31T10:17:19.02348152Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010db320\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-mclz4shl48hz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5247c381-2bd6-4d36-bde2-1113626087c1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5247c381-2bd6-4d36-bde2-1113626087c1\",\n      \"name\": \"5247c381-2bd6-4d36-bde2-1113626087c1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-31T09:58:02.878065641Z\",\n      \"endTime\": \"2026-07-31T10:17:19.02348152Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:593]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d60b00\u003e: \n    failed to create HCP cluster np-downgrade-x9g5mp: failed waiting for cluster=\"np-downgrade-x9g5mp\" in resourcegroup=\"rg-np-downgrade-x9g5mp-l2kgj8xm6ld2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/14d8c551-0a76-4262-8222-77cf6a1d6f18\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/14d8c551-0a76-4262-8222-77cf6a1d6f18\",\n      \"name\": \"14d8c551-0a76-4262-8222-77cf6a1d6f18\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-31T10:42:23.0694728Z\",\n      \"endTime\": \"2026-07-31T11:01:31.377600916Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:593]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d60b00\u003e: \n    failed to create HCP cluster np-downgrade-x9g5mp: failed waiting for cluster=\"np-downgrade-x9g5mp\" in resourcegroup=\"rg-np-downgrade-x9g5mp-l2kgj8xm6ld2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/14d8c551-0a76-4262-8222-77cf6a1d6f18\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/14d8c551-0a76-4262-8222-77cf6a1d6f18\",\n      \"name\": \"14d8c551-0a76-4262-8222-77cf6a1d6f18\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-31T10:42:23.0694728Z\",\n      \"endTime\": \"2026-07-31T11:01:31.377600916Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"11dd9f7220af8238f2ef9ee5a9c5cb972d8964dd7b3994a035437afc7f7f3335","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"1a41f9c7e28d4be4e813448eae16b579a7ac60ee08b33efd15ed967a89be1216","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"1addfb8adb980106a469cc3b5fe9b03bb44f3e6cd656845a1120a6250440fc58","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"1d7a57cdb5ee65ffb89266c52743756725522f428f1afce13dea3b7bdb071342","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"20df2b6ba32bb322126cf6122c36e047813de0db7ea7a5cf8137ef916a708392","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"2254740747b95e7502b57af21bbd058b2f09bc6d8d6fb3c239abdb1385c4275d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"2bb9725ef756de2bbca25cef6f6704ceb4d08c4852004361038a7df0bddee30a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"523b6d17b98c2ffda95f9af7a5c401b14fe6059553448e4ead4b694f0a9f1bb0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"545cc4a9fbb582d7f3746bf9174a23ac19feb9f23bfefd7ef26ff21f39001957","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"5cbe521cb7b103eac31ce83e2d45d26bc3ba849d67be55cb3edbe2fd82f6019a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"63864887a1d971437687d0c622a0475032e1c633ac916348b86960e65f25d8ef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"64f543c716edd62304f68c279b126f2aad55e5bc69d1a32c4b43dfc85a107ef0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"69684d6587e7b6d353941ba8ebeb4a372cffa88afce1255732fd16d848585774","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"6d1e234f1a78277f9a25c3930b18b6a3d1a0b1329462bce68b10dad62e917230","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"7509dc3dd13c14f7e332972aa54bd5866d49cc744c03c08e27d288c116b82464","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"78d1e58ebabdb2a96dfbae73c32f242fb8793adbf0c05aaaffca96728ed239e0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"7c444d6e8e5e42ca20dbd002273700008ee87a86b846b909f45f9e3473b47d9b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"842400b3ef0a3f5ecc338034ba9d1ca1379058fc1d6dc1c9422535ce1a4f751d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"8f3517ee40290d8188b0843d4a49397a189ea380fc365be2b10f401581714070","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"993376c4a11281edf6fd36f4246db8b66ad76f77a3d3e7e73e7dc33b5c90d087","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"a1e85439aebe8b5ced6f975c4402b396ce742c0adb62150689d9579f26c597a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"a81a4f30044c31911b6624f2b4026882651ffb15f7ad854fa349ae64e6ee8775","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"aa1c71272b3dafce965ad787dd6ed03c967304dac75630d722a97a8f0eb1a7de","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"afbd29f29515e6facd53b2c76fdaafdecd5d213a17b5401f739fdf8b15946ff6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"b30336872b5fd444e04363a76dac49ef8fe360a3399d49663dbc1396aa543c28","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"bac84b7fb8866d581dcacd4405cbd97a329ea769c878774faa06778f8a711a78","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"cc57c1146e3ccbadcdad055f15a002278a3adc35ad682c075df2bc9817cdb25e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"cf1487aacc7da5751191246b6130246032344089667301dd334a358da60e823e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"d721a9240dad38f76445ce9d9d2dbfd0c284faf1938a7d028214eb34c60e7b20","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"d8714ea0845db0e2e2bc726510d7de3a3b753b949f2c32dc2a05bbb2e833dc65","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"dda4f2b8f7a74de0f0a6419efc761bb5f73a1788a5268a3c35d5ca120e3cd4cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"df6a3a0eb5ed0897a6f9b457e97a332d7e79a583a2f6b0228f46074db5a60b80","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"df6c7c2dd2af56d660394bdb0354acecdc03bc5e7464a90b28529ea1f3a5be8a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"e82aba73d2780504045986489eec3f86cb4bff4d00a7e0dd7cdbe90ca5b0bb3e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"e9386aaf208b5343a6c85418f2a39f62f5cd16b31e1fbea5c5798f3061f8990a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"edb9770c9723ca1c44e72c42686b90f453cb45261b13824ee4169b72d6092a79","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"efccadddcfc9af2414a06989de1bd3eeedc593532e3c6ea890526979bc461878","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"f6e4dacf1ec6c6344ae470c87503c79a5a7fa05ef3bd9ff33e2656592ba44d36","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"f6ef6562a44a6c014a5645909eaf724af80a60cd9d8f9c4c7b06556e0a2f9446","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2081848906085306368","occurred_at":"2026-07-27T21:07:12Z","signature_id":"a4894824665e4caad09d676039a88f6f030faf82af7fc98d5a54bf53f764225b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"f90cae3228fe72b2427515e7c51834eebdc3e73c0b1aa6bca2abedd53a2e7578","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-f5xlqsfqjmgk\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-77tq44\" in resourcegroup=\"admin-credential-lifecycle-test-f5xlqsfqjmgk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-77tq44 in resource group admin-credential-lifecycle-test-f5xlqsfqjmgk, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-77tq44\" in resourcegroup=\"admin-credential-lifecycle-test-f5xlqsfqjmgk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-77tq44 in resource group admin-credential-lifecycle-test-f5xlqsfqjmgk, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":15,"runs_affected":11,"failed_runs":11,"run_impact_percent":26.19047619047619,"anchor_occurrences":15,"after_last_push_count":22,"trend_counts":[0,2,1,2,1,1,7,1],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-06","2026-07-13","2026-07-20"],"prior_runs_affected":17,"prior_last_seen_at":"2026-07-22T16:58:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc002c182e8\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"ephemeral-osdisk-bmrfzjw6s2bh\", caused by: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-bmrfzjw6s2bh\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-bmrfzjw6s2bh, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-bmrfzjw6s2bh\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-bmrfzjw6s2bh, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc002c182e8\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"ephemeral-osdisk-bmrfzjw6s2bh\", caused by: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-bmrfzjw6s2bh\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-bmrfzjw6s2bh, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-bmrfzjw6s2bh\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-bmrfzjw6s2bh, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000db2990\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-f5xlqsfqjmgk\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-77tq44\" in resourcegroup=\"admin-credential-lifecycle-test-f5xlqsfqjmgk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-77tq44 in resource group admin-credential-lifecycle-test-f5xlqsfqjmgk, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-77tq44\" in resourcegroup=\"admin-credential-lifecycle-test-f5xlqsfqjmgk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-77tq44 in resource group admin-credential-lifecycle-test-f5xlqsfqjmgk, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000db2990\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-f5xlqsfqjmgk\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-77tq44\" in resourcegroup=\"admin-credential-lifecycle-test-f5xlqsfqjmgk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-77tq44 in resource group admin-credential-lifecycle-test-f5xlqsfqjmgk, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-77tq44\" in resourcegroup=\"admin-credential-lifecycle-test-f5xlqsfqjmgk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-77tq44 in resource group admin-credential-lifecycle-test-f5xlqsfqjmgk, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001076b10\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-noedge-vlq6gm-8md5vxnw7l5r\", caused by: failed waiting for hcpCluster=\"np-noedge-vlq6gm\" in resourcegroup=\"rg-np-noedge-vlq6gm-8md5vxnw7l5r\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-noedge-vlq6gm in resource group rg-np-noedge-vlq6gm-8md5vxnw7l5r, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-noedge-vlq6gm\" in resourcegroup=\"rg-np-noedge-vlq6gm-8md5vxnw7l5r\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-noedge-vlq6gm in resource group rg-np-noedge-vlq6gm-8md5vxnw7l5r, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001076b10\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-noedge-vlq6gm-8md5vxnw7l5r\", caused by: failed waiting for hcpCluster=\"np-noedge-vlq6gm\" in resourcegroup=\"rg-np-noedge-vlq6gm-8md5vxnw7l5r\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-noedge-vlq6gm in resource group rg-np-noedge-vlq6gm-8md5vxnw7l5r, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-noedge-vlq6gm\" in resourcegroup=\"rg-np-noedge-vlq6gm-8md5vxnw7l5r\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-noedge-vlq6gm in resource group rg-np-noedge-vlq6gm-8md5vxnw7l5r, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2084049245760917504","occurred_at":"2026-08-02T22:50:34Z","signature_id":"b98eaa3aa284e17cebcbc58ab008cf4dbf1d5287f66b58c377446e5a8b4eaf56","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083661568511315968","occurred_at":"2026-08-01T21:10:05Z","signature_id":"0f904af658625547f2625ae558f4cb3550bcbe5b8b1552664be79ba8a249aaea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083661568511315968","occurred_at":"2026-08-01T21:10:05Z","signature_id":"7b47052217e18790be4f3e94628783ce9103c711570915db12f5cc52c398bf02","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083661568511315968","occurred_at":"2026-08-01T21:10:05Z","signature_id":"a59aab1068cc39f99f03e7d2d1f00af73481b9177294c34a56bc78879c5d6982","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083661568511315968","occurred_at":"2026-08-01T21:10:05Z","signature_id":"c26c51ac2e5a43f00675f492dae1a39c5790b351afd23da852546c9b229b60c0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083458381179260928","occurred_at":"2026-08-01T07:42:41Z","signature_id":"8546d745faaec6484b923fbde7c857adfdb77f38724b3bba967d7e8c6b1e508f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083366532309585920","occurred_at":"2026-08-01T01:37:43Z","signature_id":"40f35ad9ee71970ee1addeb8b956b42dfe6f8a7469513fecfde1464979464e8f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083366532309585920","occurred_at":"2026-08-01T01:37:43Z","signature_id":"ff3410283562ab455f05b6661eec03613dc61a18f57f4cdf6461682f200f2071","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083315168242044928","occurred_at":"2026-07-31T22:13:36Z","signature_id":"cd0e93a343d2d00b4678b9e2a78fb3f598210905965685fcf5c5c2b447b930da","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082864512502861824","occurred_at":"2026-07-30T16:22:52Z","signature_id":"883557dcd01d4fa8153b4f4d225b74622548173fe2b3838b46dbdd0e41063882","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082480222015328256","occurred_at":"2026-07-29T14:43:56Z","signature_id":"730804262eeb9b752f6827d9546c18430935b921bb62270c6128741cd6b8b3b3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082405407866753024","occurred_at":"2026-07-29T09:58:31Z","signature_id":"58dc36e2efc98292c4b5a1954f03dc497be49ff27ee959477882ae5dd1a329b0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082061073749381120","occurred_at":"2026-07-28T11:10:17Z","signature_id":"fa459fee1157c55d0acdad831257f731cc84865d15ce600e3e3798aa674e3fc1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2081795359503290368","occurred_at":"2026-07-27T17:31:54Z","signature_id":"3ab714b7767b12b0c6a64abb1c99071da5e7889809283e6a7e89020a4b113ec7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2081533282448576512","occurred_at":"2026-07-27T00:13:02Z","signature_id":"b745a349194a7286347e190afb29c1ef6f880a4513f3bf873cb0358873eb0259","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"20ec3a566a193fff5b870a1e0c66e1d6dd360c49a81eeabea9c1bf418b6cea27","failure_pattern":"failed to create service principal: create service principal: error status code received from the API","search_query":"failed to create service principal: create service principal: error status code received from the API","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":7,"runs_affected":4,"failed_runs":4,"run_impact_percent":9.523809523809524,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,0,0,0,7,0,0,0],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:264]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f05de0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:264]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f05de0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00150b9e0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00150b9e0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:264]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0002199a0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:264]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0002199a0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082864512502861824","occurred_at":"2026-07-30T16:22:52Z","signature_id":"bbcae88f9bd91ac84d2736736be0545becfbc88b8b707d6da8d236b24b32dd87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082864512502861824","occurred_at":"2026-07-30T16:22:52Z","signature_id":"cd8b18d54e848b0be1589211b07fc6892a04128fa02e494cfb003ff1579e65a8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082786541406523392","occurred_at":"2026-07-30T11:13:02Z","signature_id":"012e939c24855e55dc8fa5b4098f2594876ba83b1d2d68b4f3627221f48072a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082716084711985152","occurred_at":"2026-07-30T06:33:04Z","signature_id":"b4c17116a87ea8f85d066a721153d42c1bb9516ffa7272bd7761fa98c087615c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082716084711985152","occurred_at":"2026-07-30T06:33:04Z","signature_id":"eae8d475e571df1465546026575d2e0761f70dc0f38df88deeff37477fbb5fe1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082652897543196672","occurred_at":"2026-07-30T02:21:59Z","signature_id":"0a3cf3c8d5f004008a913b200180f87ff947a97638ff74d6a21dc4bbb03c1711","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082652897543196672","occurred_at":"2026-07-30T02:21:59Z","signature_id":"643fe0e349d725ddcfc9592ab7dcb78ba575a26497b9272fc7396e36b3b3be9a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"e0eca219c3110341e80c5a3f64ae984723abc1e1208ac11199b51c76f20de3e7","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes a GPU nodepool in a single cluster","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":7.142857142857143,"anchor_occurrences":3,"after_last_push_count":6,"trend_counts":[0,1,0,1,1,0,0,0],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-13","2026-07-20"],"prior_runs_affected":6,"prior_last_seen_at":"2026-07-23T01:42:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes a GPU nodepool in a single cluster","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:131]: failed to create GPU nodepool gpu-np-1 with VM size Standard_NC4as_T4_v3\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f22760\u003e: \n    failed to create NodePool gpu-np-1: failed waiting for nodepool=\"gpu-np-1\" for cluster \"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-nv2l5l98hkzq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d7b8561b-0d3d-47ac-a57b-83cd6af7d375\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d7b8561b-0d3d-47ac-a57b-83cd6af7d375\",\n      \"name\": \"d7b8561b-0d3d-47ac-a57b-83cd6af7d375\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T07:33:49.233891501Z\",\n      \"endTime\": \"2026-07-30T07:53:06.946835167Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:131]: failed to create GPU nodepool gpu-np-1 with VM size Standard_NC4as_T4_v3\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f22760\u003e: \n    failed to create NodePool gpu-np-1: failed waiting for nodepool=\"gpu-np-1\" for cluster \"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-nv2l5l98hkzq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d7b8561b-0d3d-47ac-a57b-83cd6af7d375\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d7b8561b-0d3d-47ac-a57b-83cd6af7d375\",\n      \"name\": \"d7b8561b-0d3d-47ac-a57b-83cd6af7d375\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T07:33:49.233891501Z\",\n      \"endTime\": \"2026-07-30T07:53:06.946835167Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:130]: failed to create GPU nodepool gpu-np-1 with VM size Standard_NC4as_T4_v3\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001d066c0\u003e: \n    failed to create NodePool gpu-np-1: failed waiting for nodepool=\"gpu-np-1\" for cluster \"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-pr6kjmvpb268\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f37e0658-bbe9-4645-ad40-9b5090460528\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f37e0658-bbe9-4645-ad40-9b5090460528\",\n      \"name\": \"f37e0658-bbe9-4645-ad40-9b5090460528\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T16:04:03.59891795Z\",\n      \"endTime\": \"2026-07-29T16:23:04.344227125Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:130]: failed to create GPU nodepool gpu-np-1 with VM size Standard_NC4as_T4_v3\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001d066c0\u003e: \n    failed to create NodePool gpu-np-1: failed waiting for nodepool=\"gpu-np-1\" for cluster \"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-pr6kjmvpb268\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f37e0658-bbe9-4645-ad40-9b5090460528\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f37e0658-bbe9-4645-ad40-9b5090460528\",\n      \"name\": \"f37e0658-bbe9-4645-ad40-9b5090460528\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T16:04:03.59891795Z\",\n      \"endTime\": \"2026-07-29T16:23:04.344227125Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:130]: failed to create GPU nodepool gpu-np-1 with VM size Standard_NC4as_T4_v3\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000657cc0\u003e: \n    failed to create NodePool gpu-np-1: failed waiting for nodepool=\"gpu-np-1\" for cluster \"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-ldc6p42cl5nn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78fd648a-f595-4ce5-8eb8-ad0b9a27df53\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78fd648a-f595-4ce5-8eb8-ad0b9a27df53\",\n      \"name\": \"78fd648a-f595-4ce5-8eb8-ad0b9a27df53\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-27T22:23:39.075545784Z\",\n      \"endTime\": \"2026-07-27T22:42:42.156031569Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:130]: failed to create GPU nodepool gpu-np-1 with VM size Standard_NC4as_T4_v3\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000657cc0\u003e: \n    failed to create NodePool gpu-np-1: failed waiting for nodepool=\"gpu-np-1\" for cluster \"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-ldc6p42cl5nn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78fd648a-f595-4ce5-8eb8-ad0b9a27df53\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78fd648a-f595-4ce5-8eb8-ad0b9a27df53\",\n      \"name\": \"78fd648a-f595-4ce5-8eb8-ad0b9a27df53\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-27T22:23:39.075545784Z\",\n      \"endTime\": \"2026-07-27T22:42:42.156031569Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082716084711985152","occurred_at":"2026-07-30T06:33:04Z","signature_id":"882351e0e6f3c040bae99057f0e95608e1726fe27b85fa498008879f0b71c7f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082480222015328256","occurred_at":"2026-07-29T14:43:56Z","signature_id":"5061ac6df5c81e5af5173fb94a2183f63b01ae6efa698c29cedbcf77eaa1c29a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2081848906085306368","occurred_at":"2026-07-27T21:07:12Z","signature_id":"88adad0cae379be8c706de40b8dfff2353d4ba3c95ca7399c0ebeef9353e9ffe","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"eb3e152dc136de977e46f73e6951fba5c50ccf266057ebb067fc33f89e7ed359","failure_pattern":"ERROR CODE: NoRegisteredProviderFound; detail message No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'HCPOpenShiftClusters'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: NoRegisteredProviderFound","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":4.761904761904762,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,2,0,0],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000507480\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-f5v8hmqt2vrk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-f5v8hmqt2vrk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'HCPOpenShiftClusters'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000507480\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-f5v8hmqt2vrk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-f5v8hmqt2vrk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'HCPOpenShiftClusters'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f13000\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-4qnnm759nd2z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-4qnnm759nd2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'HCPOpenShiftClusters'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f13000\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-4qnnm759nd2z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-4qnnm759nd2z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'HCPOpenShiftClusters'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083047482857623552","occurred_at":"2026-07-31T04:29:55Z","signature_id":"33ae2b3ca91b4bc57f90421d7bae6396376ca2aeb40277aa84a6b99500775555","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082998755912585216","occurred_at":"2026-07-31T01:16:18Z","signature_id":"a2f0e945508a29f4453f0e485ffdfc0a5dda052f2cd49414712d6c62ff904c3a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-mxqqzgl7z57c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-mxqqzgl7z57c, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.380952380952381,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,2,0,0],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-29","2026-07-06","2026-07-13","2026-07-20"],"prior_runs_affected":11,"prior_last_seen_at":"2026-07-24T21:28:09Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00152e3f0\u003e: \n    failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-k7fwk7kcgvsd, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-k7fwk7kcgvsd\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-k7fwk7kcgvsd, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00152e3f0\u003e: \n    failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-k7fwk7kcgvsd, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-k7fwk7kcgvsd\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-k7fwk7kcgvsd, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:149]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010067e0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-mxqqzgl7z57c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-mxqqzgl7z57c, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:149]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010067e0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-mxqqzgl7z57c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-mxqqzgl7z57c, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"1d4c665b4f3a2cfaa507c674a7585858146fa3441fde16f51338f0b5807aaa81","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"fe2d20bbbf0af4a5f6698a29ea9b0d315b7c445992ce9bc9ee18485fb05ceefb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"e9065928ca1d323095a060fb425d3a601fe5817c1279001c85a6c30b326c861b","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.380952380952381,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"18ada9cc0581e530fb052933f9dfaaf62842839f5e08a75f8b278747cdfb0805","failure_pattern":"VerifyCiliumConnectivityChecks failed: failed to create test namespace: object is being deleted: namespace \"\u003cnamespace\u003e\" already exists","search_query":"VerifyCiliumConnectivityChecks failed: failed to create test namespace: object is being deleted: namespaces \"cilium-connectivity-check\" already exists","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.380952380952381,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:492]: failed to verify simple web app and connectivity checks for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000e0a078\u003e: \n    VerifyCiliumConnectivityChecks failed: failed to create test namespace: object is being deleted: namespaces \"cilium-connectivity-check\" already exists\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:492]: failed to verify simple web app and connectivity checks for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000e0a078\u003e: \n    VerifyCiliumConnectivityChecks failed: failed to create test namespace: object is being deleted: namespaces \"cilium-connectivity-check\" already exists\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083804810854797312","occurred_at":"2026-08-02T06:39:16Z","signature_id":"a8b8a23ec15e3d3fb68c3959c81e60c4fc87875b9d9eabd555cfbf8c4b402ea8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"10aad24726b1a7f164905b0ac4ea1b1a11bc7afc424311924625d3b99dedbe4a","failure_pattern":"add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","search_query":"add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.380952380952381,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:269]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002793860\u003e: \n    add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:269]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002793860\u003e: \n    add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082171433848410112","occurred_at":"2026-07-28T18:28:49Z","signature_id":"9f6788db99878820c6d05b8aa40566d5fd1eec43ee6c28b743ac410dec4becaf","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"7207c83a34a191366f94a6a113fc9a5aaabe50fb5e6befcd21b6e21322f73a87","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.380952380952381,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083112969008320512","occurred_at":"2026-07-31T08:45:01Z","signature_id":"d712d5ce17264f0ef550496665e2a7bd7d67dfac479ff00deed5bcb5e610f7c9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"ecba9c70d623dbfd01226f49890ba9397974dd3b8819dd0355ce6670accfbc55","failure_pattern":"expected \u003ccount\u003e ready (and schedulable) nodes, found \u003ccount\u003e","search_query":"expected 3 ready (and schedulable) nodes in node pool \"npupgrade-4-21\", found 4","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.380952380952381,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:232]: failed to verify 3 ready and schedulable nodes for nodepool npupgrade-4-21 after upgrade\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002b03f00\u003e: \n    expected 3 ready (and schedulable) nodes in node pool \"npupgrade-4-21\", found 4\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:232]: failed to verify 3 ready and schedulable nodes for nodepool npupgrade-4-21 after upgrade\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002b03f00\u003e: \n    expected 3 ready (and schedulable) nodes in node pool \"npupgrade-4-21\", found 4\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2082716084711985152","occurred_at":"2026-07-30T06:33:04Z","signature_id":"7007959012ccc8a940760507dc35c9df32589e0d43616950347a0bd0afe52668","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"221cb0b46f3bef91cff2ee06c068eca8fb3646ef7eeb7ed9cd39ae59c1446e81","failure_pattern":"list nodes (nodePool=\u003cnodepool\u003e): Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"list nodes (nodePool=npdg-4-21): Get \"https://api.r9l2c1j8d5l2w0y.xhnn.uksouth.aroapp-hcp.azure-test.net:443/api/v1/nodes\": dial tcp: lookup api.r9l2c1j8d5l2w0y.xhnn.uksouth.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.129.184.62:37315-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.380952380952381,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:621]: failed to get node release images before downgrade\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cc1140\u003e: \n    list nodes (nodePool=npdg-4-21): Get \"https://api.r9l2c1j8d5l2w0y.xhnn.uksouth.aroapp-hcp.azure-test.net:443/api/v1/nodes\": dial tcp: lookup api.r9l2c1j8d5l2w0y.xhnn.uksouth.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.129.184.62:37315-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:621]: failed to get node release images before downgrade\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cc1140\u003e: \n    list nodes (nodePool=npdg-4-21): Get \"https://api.r9l2c1j8d5l2w0y.xhnn.uksouth.aroapp-hcp.azure-test.net:443/api/v1/nodes\": dial tcp: lookup api.r9l2c1j8d5l2w0y.xhnn.uksouth.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.129.184.62:37315-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2081598121950842880","occurred_at":"2026-07-27T04:26:06Z","signature_id":"541d88e21b4cf0911f55c0d2ddbd62d23524d5ed0e3b07d1433b70b4a76e35a7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.380952380952381,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-07-26..2026-08-02 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-29","2026-07-06","2026-07-13","2026-07-20"],"prior_runs_affected":6,"prior_last_seen_at":"2026-07-21T10:41:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:186]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-dqrl4d\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ed73e0\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-ch254.apps.aro.n0v1e5o4a9v5f2y.oymy.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:186]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-dqrl4d\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ed73e0\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-ch254.apps.aro.n0v1e5o4a9v5f2y.oymy.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2081747780430204928","occurred_at":"2026-07-27T14:25:20Z","signature_id":"9a5ed036d2527c381910d54ab1b1ab24a0641008a56e3979456f337317925579","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}