{"meta":{"start_date":"2026-08-18","end_date":"2026-08-24","start_at":"2026-08-18T00:00:00Z","end_at":"2026-08-25T00:00:00Z","timezone":"UTC","generated_at":"2026-08-25T23:49:55Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":587,"failed_runs":454,"raw_occurrences":1452,"matched_occurrences":1388,"runs_affected":390},"rows":[{"environment":"dev","failure_pattern_id":"4b645ca11afad5b79ec9dae25eb364818cdce5c3fe97c8a6de8581921ac37e5b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":128,"runs_affected":27,"failed_runs":27,"run_impact_percent":4.599659284497444,"anchor_occurrences":128,"after_last_push_count":90,"also_in":["INT","PROD","STG"],"trend_counts":[29,26,14,33,40,14,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":84,"prior_last_seen_at":"2026-08-22T08:48:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":7},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":7},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":7},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a default cluster and verify KAS is accessible via both public shared ingress and private Swift networking from worker nodes","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS, private ingress, and private KeyVault and verify all are operational","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP creates on-demand backup after KMS key rotation completes","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-nzmkwtbzb8xk/cluster-candidate-4-21-k47dlw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00011d300\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-k47dlw: failed waiting for cluster=\"cluster-candidate-4-21-k47dlw\" in resourcegroup=\"rg-candidate-4-21-nzmkwtbzb8xk\" to finish creating: GET https://rp.j0629888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e224492a-2630-4a37-b5e0-7ef00f6cd9dd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e224492a-2630-4a37-b5e0-7ef00f6cd9dd\",\n      \"name\": \"e224492a-2630-4a37-b5e0-7ef00f6cd9dd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T17:34:09.560283017Z\",\n      \"endTime\": \"2026-08-24T17:53:13.775593008Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-nzmkwtbzb8xk/cluster-candidate-4-21-k47dlw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00011d300\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-k47dlw: failed waiting for cluster=\"cluster-candidate-4-21-k47dlw\" in resourcegroup=\"rg-candidate-4-21-nzmkwtbzb8xk\" to finish creating: GET https://rp.j0629888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e224492a-2630-4a37-b5e0-7ef00f6cd9dd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e224492a-2630-4a37-b5e0-7ef00f6cd9dd\",\n      \"name\": \"e224492a-2630-4a37-b5e0-7ef00f6cd9dd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T17:34:09.560283017Z\",\n      \"endTime\": \"2026-08-24T17:53:13.775593008Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001002c80\u003e: \n    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-fvxb2h745mmv\" to finish creating: GET https://rp.j3442688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/927301b0-80ab-45c1-a577-7613e32d2b80\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/927301b0-80ab-45c1-a577-7613e32d2b80\",\n      \"name\": \"927301b0-80ab-45c1-a577-7613e32d2b80\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T09:32:34.244455805Z\",\n      \"endTime\": \"2026-08-22T09:51:43.68061461Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001002c80\u003e: \n    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-fvxb2h745mmv\" to finish creating: GET https://rp.j3442688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/927301b0-80ab-45c1-a577-7613e32d2b80\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/927301b0-80ab-45c1-a577-7613e32d2b80\",\n      \"name\": \"927301b0-80ab-45c1-a577-7613e32d2b80\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T09:32:34.244455805Z\",\n      \"endTime\": \"2026-08-22T09:51:43.68061461Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009fd080\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-f7hhtzrszbl9\" to finish creating: GET https://rp.j8029824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/edb9f485-31ea-4549-83fe-b228b1e92d7f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/edb9f485-31ea-4549-83fe-b228b1e92d7f\",\n      \"name\": \"edb9f485-31ea-4549-83fe-b228b1e92d7f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T05:37:57.735593463Z\",\n      \"endTime\": \"2026-08-22T05:57:05.414598604Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009fd080\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-f7hhtzrszbl9\" to finish creating: GET https://rp.j8029824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/edb9f485-31ea-4549-83fe-b228b1e92d7f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/edb9f485-31ea-4549-83fe-b228b1e92d7f\",\n      \"name\": \"edb9f485-31ea-4549-83fe-b228b1e92d7f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T05:37:57.735593463Z\",\n      \"endTime\": \"2026-08-22T05:57:05.414598604Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"9806f10f5f28de5b459cf7d30f0fa456f7ab916323bce3ad88103d4bb5b0c102","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091085161763442688","occurred_at":"2026-08-22T08:48:46Z","signature_id":"96ba3bbe47b7611d618631961d3c5569fbac980522d504f0be58ba9cb697daf6","pr_number":6640,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"0624e47ebc01259a86393f6b94bf7768700270439ec6340fe1971fa2e93fe976","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"1452f7adbf81aa74b093ca45c0fa8ac97627c68d98fe9e5920a5f92b6a914584","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"2e4c8f866988abe4f3fbe1b08e877e0cfcec027a2f7cac659c081bcd21149734","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"59c1e1347bd427d8d4783d93b76ae2027a93d4ee3213b2ac712d69c81a638899","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"5d6538d0941a77db5c74d2366ce1aa2770649359b913f22f6200ba4597a39ac4","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"63db49341df819db34eca5b5ac261b93de3b59ce36b9b3d9cc9b869199075023","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"736aa2a9bbb3e9cffb2dbbf9d439c8108f35bc75e1a24329e387a56a5967bf82","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"b07ba0edef0c9c36e19410fbdad5237fb04ec9614b2c3154f0326e6c02187e76","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"dba72741ac72f22b446e90bc10eae8af85de2d4bc3c2708c1e311e42dca76710","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"e20c717ced12823cf99b365d56e4c2a886f13b4deb5bcd5d3e3e323a6edc8827","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"f60c643736c76a179834c7fa2cab4c02555094c1b8f19768ccb5a570ec769e60","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090960143322714112","occurred_at":"2026-08-22T00:32:00Z","signature_id":"1acede20c12184d185bc2dc16bc7034c0bbd254bcd41cbb5575e6ffd0a010dca","pr_number":6639,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090958470403592192","occurred_at":"2026-08-22T00:25:22Z","signature_id":"b4f721e5ea0f91bfbe7985abc52715ae48810e5fe0ee1d0cb9f7f8d919e916b2","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"20d6beb16979adfe257392f9563b7f0ce21b67ae5819ce4215adfcbc2687c77e","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"24bce759fd4066d1fc588db89a49afde37cbc6f44bb5f6aba66d9a685d775c4d","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"38b585818bacd259d2b2d3469bb56025bdb56c75d07786458c1a90fc46168739","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"4400d5437d5e17e6ebaf9c42924e7b9494fbb3607ad67e3b76de4540150f3be5","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"558ce75781a063d32559631db7e9f63baa11cbaa0afcc8f29782f0c9b26502d8","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"612bc332a79ff67dcd08cf1d8fb4be9d1085474e2e3993b2e0eefe78d68f6752","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"7a0a1e4f4e85d34d06ec7c677ae0c840a32378750971186a367e902850bc07de","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"80fe67f4f734e23425e71a187d1fa480f02fab99935a1c2267e4f7e705acd284","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"8e6bf6dd5a581206dcb87e48ecf50a24fa31dd1fd66ba0ba42db56ffebb2cd6c","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"a6df34000b844d5b45630664d5b48bba8d389682ea434496b73521132685d440","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"a9c19efdb9bd31e434c2b860e318675d4fe13ce36666fea13eb0e62e55f757a3","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"ad8dfe8ab83d51da65319f6aaa59b17ad6fd532c019b7bd9c946b6f18083edc6","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"c4bfe25397b8b2d437eae4569a7723b855e9ee12034118a652c42c8be6cdba5c","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"e26ff3c2c390f7d5fcc4d0231fd0d98319d7438d1526b577703513ffe3587558","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"e624b5aa4b7740b1ba645384a0a92d298c7cf268038db0a9142479ea68b268db","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"5314a86bba0bb9f12a61be0bd6791fecc87d5f8d00fa3914bada32d665291e4a","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"5abca12243bae993f07f4e4e63a4742fb8d2e7c8e46fcd64385eec5fdb894b7c","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"6210f5fef10139baa9478a3153593fb949e25d7abffb54ffa7e12322cda4088b","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"6f0420fd84447d47b60aa6ff6601d0849f8dbf938905c55ca4009f1b4d455a72","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"98685d5e8b443df38e924eb75184bfca6ba5d2dd3e69379f57c60863b5eaae87","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"9db5b43574c9db1eac22e7d04f83f6d559414b29d58cf3741819a9aba0e41e9d","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"a57b62065dbf4f89ca41062220a43ae387245fca845dcefa7c5564685c21c452","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"b47de51b0f358050d51a602d396e7136be355a35482bfffa6484fba844fcbc14","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"c000eccb673f0180f2f3508c516881b27fe7b2013749e38d8cd16f2c313d4a06","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"f086b4934e2f69cee46a0807f947e11514153d16a601e33d6288b4153bf2445b","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"fd35d176d4a2777f7fd1120d1d75ce1502e674361791c7c82a8b9d0642a8b75c","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"014b0124a0340b2d0058f92137c1af01c74b55c601a11f17b66726689e079b7e","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"060e9895c1ea9b171d554719ff54ac5c009daa12670fadcf5136df441b33912d","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"65a5ada1bbaf672403f3ea6bfd30e59064f627383b60bd9b883c7505c21cfa14","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"6c0b36b89e89d4e3268ffb83a5695f5ec11294751941582771219733b9d66cfc","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"7ce3cbe97b9c16950bd30a5c6ecdc0d15277684addd5eaeaab7d9097b895113c","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"b3e2ca6406b7d0c6d46b39472954b647cf1e029d807df155f8090c7730519eb8","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"c257cfe9f4f89612bb73d7dcdf0bea39effe21ea7a9e7e339d581803c506f2a3","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"d1938d6322253f9c816c3bf43e93a7b6a173eeb9c6980a4b792ae7965b363f77","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"de3b527ad7839cd28018fb7bc4ecfad91f38d4772c6b2e873fde7ed665f9392e","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"e0211790082612284639b917f2152b2d4179b71035c60ed504f9fa103648d359","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"f9e689ec1684b8942c33d33f9c825e3407a1998990b5718a09ae5147fa362099","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"09094648223231bb9dbde764787354bcdc986a4c2d7c90b0773b64e1f0744693","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"430ca5b576d22528c6770eacf2a6c8f97dab34d25d271c47fd31c6ae0cdf9b69","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"b709b68e31b4d438b7979bfe9e931a7b3e4f89ed2b8708048c02a5b68f58e960","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"04bcd6a06d29057b10be12e04816d64f01f1530b8ecf366646c79fb09d1cdf11","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"246c803b1e707a889593bb8ea446899c00d7d4d657aab923a9e9edfe4145c115","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"5bea59fcaf64de56a61dc5fd611024a94b77bcc55b77b15df0fda9ff595af14d","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"5d468fd6480dcdcd43dc7d63806754d69878da96ff0fa95f52d664649c52df57","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"8c4a7fbbe76c05848c2f39efbe44879deba3e853f5ad0376ee17a8c4078e8717","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"9c130e71e529e7927b1ef5616dfea401f80d02d3d228f55ea0a3620ca0df430b","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"c2a1bc6923afa666a0125d5314a52ffcd5da96f4286af2c534bd486ed9e26327","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"da3035a89dd9906e064acdf1fe56dc3f4a9a46899ce1afec10fcf46eeb40e518","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"dcb25a94761633dbb22f75fbad3c9984bd81a1f744d278223f859c52de991ad2","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"dffa1f46e9b78e5e92a4bf03da03eebd9cae7182e52521b5d9053ab0ee31762c","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"ed778a116fa87cc48eef03343ad5455868c45c5e4eaf677d04ba06620101bcbe","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"20ef09c3c97235c2e9f93c63c6114435707b1b4b0ac57f2871c9b9d4a0196473","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"43aac46da268d3089c276a3a7631b3948dfbfd971513e242bb5454c0ec6d3645","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"539ccbc9dbd92a40a1093e109cc7a09e024aedf2ea6c2db8584640c0a28c1013","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"5b42f898212fc24fb029dec13a60a0660c04dd62977d06a96fb0ecfe94159be2","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"601387522ed142b50871a3c00bc33122ab7c4cb9f2d93fc05df3d55cf90311af","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"7ee8f0811965f3a76e4945f432b1a1c826430f4d21048c68406255e9480f033c","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"be2cef2f811fde74db3929e22ea469291a511bb9389c23ca64d680bbdd1109ec","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"ce6a1cf5235a0c582fe2a45b9ca1098300fd89148eeebdd597312a281ed9b8da","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"d09b7d40ce5ea9e928c47d69506c3e959ef1199d4abc5ccde3d3ee558fbb4613","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090497381098328064","occurred_at":"2026-08-20T17:53:09Z","signature_id":"8eea666df51a57e86a1aa975db5a69225adbc526d8483f639ecc78381a317867","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090465920358027264","occurred_at":"2026-08-20T15:48:08Z","signature_id":"fd4d79833e2c9a4f42e2fba91ee43b4d71c5bda4c82416b82d7173763d24d9cf","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090405439429677056","occurred_at":"2026-08-20T11:47:38Z","signature_id":"c32c7397b025363d964e7dd9b9c594012cac3c74f81e392ab4a145c1c243c792","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"00cef274ddffb2732acc632cc1b182ec59fb565f02720c8b67051535ce772f83","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"2f55155f7b22fa74558cdf44004f4500ffdc59a95b789699899d148245d4d673","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"748b6ba9d3c5cd5a2f3579256ea3a7348e7ff1d2c5d3c8e2870e22efa3a6878e","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"8a9137894c721007ad4bf245bb688d32d3743ef19d01d7b13ceab27d2d7bd6db","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"a94744931a24c357dec9cf5c8170c369eabfba697bd1a7a95d81dc746805f80b","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"ac7c80c51e0db130adc148de4ff3c42257383d067d073129335c97f399ab326f","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"b54898cff87ba825361731cad5c3d16a6aa7cf7febfbc0f3c0924bf4bf06be42","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"b97950402c2638225fdede0ae11cc1339025f1866c657ebfbd50ae5666f586f0","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"d000285be635d8d22977be53c3a079b5b8b94ed11e8d9d6b46e9e0ec678b4a9c","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"d8c2c2dea9d1fccad931eccb2040ed9768b2ec564e3e703465698e8ec24f3692","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090191599463043072","occurred_at":"2026-08-19T21:38:05Z","signature_id":"47752b32579446bf4bfc4d096c1e5033967d1b7f583b8cbd0d426ad63ef21752","pr_number":6608,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090133175878553600","occurred_at":"2026-08-19T17:45:52Z","signature_id":"6f47ddd82e439f89e71758068a026c71acf578ab4e0cfa1e4da4103013b9a768","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090085264507539456","occurred_at":"2026-08-19T14:35:33Z","signature_id":"d688cab4cfde23219f6001dad334c01cc37a31b4faf222dcc80630c4ba631dcb","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090085264507539456","occurred_at":"2026-08-19T14:35:33Z","signature_id":"f316e727f1813330d33076f4a020c08fdc5b05e75a46acf94630a96035d90fcc","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"08288e83cb5aad9a4943f319e2ddda651d5ffe0ace3f2d0d802c3d560cdd2ef9","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"2a405dc6a6ba24177e78ccdf06b8833de0334419e3f0404f8301d9994587b574","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"516a9fb731b4f124da567dae4c2c2ec5a10033fc9b93c35c3ed7afe5b88c39fb","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"559acce7c1322e003687e061addb4f08d87fa4fe5fb407f3748561a319b962b1","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"5edc6d0663ee0be14c9cb5d0ec6ac6990a7c5b0ab006bbf59dec57df7c41383d","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"8285d0d5b2c0708cff4391fb99a39ffc99f1f8527c748614b04ae6c318ae8193","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"9f068f29f402d90a1d62ce54b8fd412bf10df785fb8156d8efe3ed8c5d9ebd81","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"e4969ed09fecf108227e0a76e171ac4453c2b4a62d9b74ef9ddb916c6b9dc192","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"ebe92ac8ac5fe50b70fcb288bbf69fbcdf672537e9a80d11d7a2de1dfbd2bac9","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"edbee3da9aceb6c190f8576e1b5366014cb172c4899e5874bb8d9f28ed57eec9","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089829163719462912","occurred_at":"2026-08-18T21:37:54Z","signature_id":"2428d8392fd4df70b709ca44873818ca85630718c317c98f8b9f713e2db130cb","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089829163719462912","occurred_at":"2026-08-18T21:37:54Z","signature_id":"d8bf2c3aeff4886e86d5669a44f4762f53e5eac0b7fa45db22c6138374017402","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089759198756737024","occurred_at":"2026-08-18T16:59:53Z","signature_id":"a2b7163e38954e647861e0b399524fcfd25f9f0eec428ca061bad52437ce1f68","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089759198756737024","occurred_at":"2026-08-18T16:59:53Z","signature_id":"b34ddc9eb703244f33af5d67b2d3ad509fe1914833aace5544a339e8af75c024","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089754858566455296","occurred_at":"2026-08-18T16:42:38Z","signature_id":"e517fe79fc6a9839f242fa59854487b41bea1fdb8ee37d04e651e4071a686562","pr_number":6553,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089753382901256192","occurred_at":"2026-08-18T16:36:46Z","signature_id":"3ca6d07b85594d42af302ab67d81529744f7099e7653a2840ee006794fcf5bde","pr_number":5900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"10a1ef5a1606f1f357b2951782e7f7cb79ef48205cd995207761315811c9403f","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"173657bdb106277aeae2a09d99b5cb1725c84d15fc8599975752d18087cc5770","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"3d87dad2c9d961c885a16f21ff019d4f45e9e09dc9726869d1b4896eec06350c","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"545b0a8b1c6348aa6f3361f2850594b9ce7a7450ee8f1b3acc595bba5c0fca21","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"57a33e2f071e18ac60014fa284a68fa66ef98d247a604ed73be8bb819c5aa38e","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"5e906a55cdea4d0187201a115b8d39c0609835ee5a3dfc81e9d5b7978fc5d201","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"64e10723887be24d8d535a46ef0bb9c4ac13e1e1f9d44611e25fd57ec09810c0","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"6b1a4bf217836c8ba644d9b912197bf3eb1257f18e9cc2971ab4cd27b10f4c65","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"93a305521066450e13149f8691a6ca8827805fab4ed9a68964cbea965bcc881b","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"97029ce1e5f07005340d9e0dd6e062986be0f648e7d0ea6911f45b9d60e36b34","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"afc03bedb7481cb76e7e3f165e34395b4318da3d63feb907bb640ae5f6157d03","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"b035d82a7d898ee3ef425a158cd91c9df1a1807fd11e22761080866f0219e0bd","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"c190c0cad31aac85184c0fd4119074e5409690ae23b21ceb7665a0e8244e02be","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"d92fabfdc0edb02f1fc4a8044563f8cc92d446055cc249538620f4e57b6ea837","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"f5dea8090edb8c5d370ebca5c0c472ce993f062e384c087128b868d43b01acef","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089713360550498304","occurred_at":"2026-08-18T13:57:44Z","signature_id":"10844cca31f8b35780dfd5b6708825e47eeac4379e5d63893007bde0469821ae","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089713360550498304","occurred_at":"2026-08-18T13:57:44Z","signature_id":"8a4f47fcb169200f5695ccc4465d56ba81a0387ebc949e190e5890f2d40b62dd","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708056525934592","occurred_at":"2026-08-18T13:36:10Z","signature_id":"12275746960d8093410a30e2792b23ea8063744fca609b8e6f8c02bb1aca165a","pr_number":6585,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708056525934592","occurred_at":"2026-08-18T13:36:10Z","signature_id":"de2643f14d51038d52491b6ef992a840366a59babc95698a840cdd28829e4954","pr_number":6585,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"c752f4ffbfd79b496d247ce40ae9cbf4f9b6ac42de61435e744772e3df1ac9c9","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":109,"runs_affected":45,"failed_runs":45,"run_impact_percent":7.666098807495741,"anchor_occurrences":109,"after_last_push_count":215,"also_in":["INT","STG"],"trend_counts":[33,43,30,18,17,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":168,"prior_last_seen_at":"2026-08-21T21:25:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":8},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e8af48\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-mb2sm98hvbm8\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76928be7-64f8-4a4c-85f6-cd6b790148b9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76928be7-64f8-4a4c-85f6-cd6b790148b9\",\n      \"name\": \"76928be7-64f8-4a4c-85f6-cd6b790148b9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:14:43.826676557Z\",\n      \"endTime\": \"2026-08-24T15:38:50.051171369Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tm25j05l3q2f8a1e83cjmn4lda8g still exists (deletion dispatched at 2026-08-24T15:14:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e8af48\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-mb2sm98hvbm8\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76928be7-64f8-4a4c-85f6-cd6b790148b9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76928be7-64f8-4a4c-85f6-cd6b790148b9\",\n      \"name\": \"76928be7-64f8-4a4c-85f6-cd6b790148b9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:14:43.826676557Z\",\n      \"endTime\": \"2026-08-24T15:38:50.051171369Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tm25j05l3q2f8a1e83cjmn4lda8g still exists (deletion dispatched at 2026-08-24T15:14:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000efa360\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-hzr77tbdkrwt\" to finish deleting: GET https://rp.j0088320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf6ae19-b7a5-40cd-806b-5002e0ccbaca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf6ae19-b7a5-40cd-806b-5002e0ccbaca\",\n      \"name\": \"9bf6ae19-b7a5-40cd-806b-5002e0ccbaca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T22:23:11.170362844Z\",\n      \"endTime\": \"2026-08-21T22:47:12.159444276Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sb7tt2c2j1a6r0ch04519gisqcoillq still exists (deletion dispatched at 2026-08-21T22:23:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000efa360\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-hzr77tbdkrwt\" to finish deleting: GET https://rp.j0088320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf6ae19-b7a5-40cd-806b-5002e0ccbaca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9bf6ae19-b7a5-40cd-806b-5002e0ccbaca\",\n      \"name\": \"9bf6ae19-b7a5-40cd-806b-5002e0ccbaca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T22:23:11.170362844Z\",\n      \"endTime\": \"2026-08-21T22:47:12.159444276Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sb7tt2c2j1a6r0ch04519gisqcoillq still exists (deletion dispatched at 2026-08-21T22:23:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00182a330\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-p78v2jfc6wjs\" to finish deleting: GET https://rp.j0835968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a8467ce1-61c9-4bf1-976b-a21d431ed9d8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a8467ce1-61c9-4bf1-976b-a21d431ed9d8\",\n      \"name\": \"a8467ce1-61c9-4bf1-976b-a21d431ed9d8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T22:56:57.039348438Z\",\n      \"endTime\": \"2026-08-21T23:21:07.00564298Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sb7rc59dodo08pmvlbds9o0qolurven still exists (deletion dispatched at 2026-08-21T22:56:57Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00182a330\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-p78v2jfc6wjs\" to finish deleting: GET https://rp.j0835968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a8467ce1-61c9-4bf1-976b-a21d431ed9d8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a8467ce1-61c9-4bf1-976b-a21d431ed9d8\",\n      \"name\": \"a8467ce1-61c9-4bf1-976b-a21d431ed9d8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T22:56:57.039348438Z\",\n      \"endTime\": \"2026-08-21T23:21:07.00564298Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sb7rc59dodo08pmvlbds9o0qolurven still exists (deletion dispatched at 2026-08-21T22:56:57Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","signature_id":"53ff086f26d6ee86f70c417c33372b3bc3883205891aacb5f42319d457fb9cf5","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090913249020088320","occurred_at":"2026-08-21T21:25:40Z","signature_id":"bb2b90e4f29b12a3abf513339d37627508f6b4fcb179c3da34fa33dfad9d2de1","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090912033770835968","occurred_at":"2026-08-21T21:20:50Z","signature_id":"8f4a0db4076e1cf40b1861e6f93b93e2128a0306906fcdbdc7164a522fed9fcb","pr_number":6651,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090912033770835968","occurred_at":"2026-08-21T21:20:50Z","signature_id":"c57fa18006932cc4fa1a5c28791cae06cbdd74d294833ef29cd896320113cdf5","pr_number":6651,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894675127832576","occurred_at":"2026-08-21T20:11:52Z","signature_id":"23ec657957baf3f24205544e835e35c4734001a72a2c3076d95cb8519ffbd14d","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894675127832576","occurred_at":"2026-08-21T20:11:52Z","signature_id":"9deaefb6491c44a515e441c706ff3b3b31a8258d9f6b976bb9e6d307cce036a7","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"138c6eb6ec4cfb0f559faf8b27491e176a7e3689afd15778fe3c46693d7cd281","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"ef819eba9a8e82676e750c398e22e3fc87dc87851fbe6f06f07fe711d52b6a42","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"f7382c924eb5a3492d2ea34a9d0cd2081b3398e3a83bada4556de6ed77fabd24","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090676319984553984","occurred_at":"2026-08-21T05:44:12Z","signature_id":"0cc8318af89f3897857545e5af36e051cda23e38362afc2f37da5c1480303070","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090676319984553984","occurred_at":"2026-08-21T05:44:12Z","signature_id":"402189eabab67acb37888ca48022e0b174cbf93a51611d0579b67a44a25ed753","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090676319984553984","occurred_at":"2026-08-21T05:44:12Z","signature_id":"83e5dcc44bb56342cd157a1d9a58d7061fb36881f41e5de18f10c278c5865f9a","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090676319984553984","occurred_at":"2026-08-21T05:44:12Z","signature_id":"a4dbdef4b3f336d8893721cb7f8b34aae93f20865aeee833ebec5277bd6add9d","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090660118717796352","occurred_at":"2026-08-21T04:39:49Z","signature_id":"c122f69522e512ddd46dbe33faa5fae6273587778e6ee59c2db8a19c13119aef","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657643948412928","occurred_at":"2026-08-21T04:29:59Z","signature_id":"1a32c1a95efe7a29e295ccd982d2b44cffbba78df2bd1aaf0fe98ce1680dc016","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657643948412928","occurred_at":"2026-08-21T04:29:59Z","signature_id":"4cb7505b78171aefbb12d1f02e1c343144f3ebcfd268c6c0359c6fbf21922214","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"e4927c48eeabe34bc637b2643be12f8daeb52ec7c7626d29784709d3ee69d6f9","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090643626764800000","occurred_at":"2026-08-21T03:34:17Z","signature_id":"c6f251e281ff40fad947b6e76150b6760e590c03389601e036c5186f4a26607d","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090587456947621888","occurred_at":"2026-08-20T23:51:05Z","signature_id":"2a86e1a5bc401c03003c5845b248e027c44091ff5733e9efbd5b9a4bae152e77","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090587456947621888","occurred_at":"2026-08-20T23:51:05Z","signature_id":"e74f713f7a408119d04d974b97bfa35814e064db65c1703843c725496b2f8453","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090580629858029568","occurred_at":"2026-08-20T23:23:55Z","signature_id":"848724995a1498efc32129c35dffe5f7984c95f5e05ad1db60ad5c24c1e46924","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"6cbf440bc241ca9f30112587152f829d7ea81b8499f42d4cbdf631a896f8d037","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"6dcb427140d419621d24ab58c4f01fc95f92d9e6a6a74b94f3ccef1dc39e23bc","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"d28512cbbd0640af93827eeaf63af32a97f35479064f9bacc1353f4b29ddd213","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"c4469e6bfaef3e79efc3520cfb16a3b01f272605bc2e4ebf424fbbe45f9f6f58","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"efb76380c356925b35cc743b1695d2943cf8e8c4170a58cb685a9f26127b55d0","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090473213871001600","occurred_at":"2026-08-20T16:17:01Z","signature_id":"617f17096ddaad86b01334d98d8a379bc6dd44009a8565f420ca7b061da140c8","pr_number":6621,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090473213871001600","occurred_at":"2026-08-20T16:17:01Z","signature_id":"bd4d29c5320aea71d9f4b595ff9a6f126321aa2f9bec81d58bb337e616cb3347","pr_number":6621,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090443849187987456","occurred_at":"2026-08-20T14:20:26Z","signature_id":"7aa544050c69c6477ca85a4c3af70f1dd4a2e874e303c1b6b9968230f6fdbba1","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6589/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090373215313465344","occurred_at":"2026-08-20T09:39:46Z","signature_id":"70706e0e8a21dc34bb78771715c8487aa916955fd148bb428a78d36cc02e8913","pr_number":6589,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"386fa1316771c34bf0673e90f0051a18d6c5b4d7928e72e6319a9e2b346bec51","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"6cf80e4f74032eb95372cfafe9faa70d564caae28c956b32fc461ed70bfbbbe5","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"7b94f144d48f01b8a2dffdde46780cdeade986950e1c5cb7f10e8de5f6208290","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"080b1980af2148b254ebc364ec074b8b45a3913630fbee6ed39ba719bddb6412","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"c4ce75502913f8985263519e26b78cede5d045fc2b6518bb4355d5eb0f87c381","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090249792243372032","occurred_at":"2026-08-20T01:29:20Z","signature_id":"64b35d7107bf8bba32f177b18c07e632a8b85536f55028f9e8786e6cba992f6d","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"e56d745d1631f725a61b02fb97814b4427228a287c46c699fe0e8e754c2d3097","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090126361980047360","occurred_at":"2026-08-19T17:18:51Z","signature_id":"177b39dfe4594f7e6fb06a6f7750f916b44fd74d75da8566dfc2541c1b7da113","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090126361980047360","occurred_at":"2026-08-19T17:18:51Z","signature_id":"22ca9334c873ccba6a931cdcceec158397b54365ca7d2223fd061a1747a742d4","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090119905717260288","occurred_at":"2026-08-19T16:53:12Z","signature_id":"c3dcb5c0b950d390430a28a8362471db978e5fa8025bdacec6e0307a4e0ffa13","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090111171486027776","occurred_at":"2026-08-19T16:18:30Z","signature_id":"4963cb1cd863c9546d3acfa468246c7a65ea1896bf8c3d2e866fb6d92513767c","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090111171486027776","occurred_at":"2026-08-19T16:18:30Z","signature_id":"66b05bcdba707719a29ade3a38a592258173565a7afd218650cb24d16360c142","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090111171486027776","occurred_at":"2026-08-19T16:18:30Z","signature_id":"a67162d2dd9c478c5a34a2c5a352c6518959da807cd853356ac9ef9bbb9355f5","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090087196852752384","occurred_at":"2026-08-19T14:43:14Z","signature_id":"2e68ce4465ab7b4784295bc0f537a0cc12ab0b9eb033768dfce3b9397fbabc37","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090087196852752384","occurred_at":"2026-08-19T14:43:14Z","signature_id":"991fa55c1cbf281c8a6a21502a56f3cf0243c994a46b84c3c129c86d08500e94","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090087196852752384","occurred_at":"2026-08-19T14:43:14Z","signature_id":"d1f00d98b96fc43cd8cd04fb3f710f5c58d8e5c745304cfa7bed7ca8f9cf798d","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090085264507539456","occurred_at":"2026-08-19T14:35:33Z","signature_id":"c7e64cdff7fd089d3fd2dd3b4ae01e0ae9578c12c9f77faff280f06ac6241164","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090085264507539456","occurred_at":"2026-08-19T14:35:33Z","signature_id":"d667f20e991a417af83dd002ae338e794d5ed667f5e3b1ee7b027b63a16e69a5","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090079394612645888","occurred_at":"2026-08-19T14:12:08Z","signature_id":"45c9a8bb121ddb135d411d85d961a4f1313b74d1f7b28d3f73e8aa3e30aa988b","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090079394612645888","occurred_at":"2026-08-19T14:12:08Z","signature_id":"a7de2a9cdec6164a3b14253e84d4f4dfe750c11eb696b14caf4efcef018b75d4","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090079394612645888","occurred_at":"2026-08-19T14:12:08Z","signature_id":"ea68d0fb40ccf06f806678a58f24d5aa4bd9e216ba376d8f98410f8788f2f8e3","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090072497390096384","occurred_at":"2026-08-19T13:44:49Z","signature_id":"2bfd6cd29638b751942a8e78dac53159ec41dd94cbed6747c22299cb4f4bb07c","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090072497390096384","occurred_at":"2026-08-19T13:44:49Z","signature_id":"4cab3b4d6ef0a8436b0f34860e699e4f385289e25548e1ad56907806d84b53fb","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090072497390096384","occurred_at":"2026-08-19T13:44:49Z","signature_id":"6ccf4df41b29745ca9abeed380bd011941d1644aea3e42ab4bd657f5c0fe34e8","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090072497390096384","occurred_at":"2026-08-19T13:44:49Z","signature_id":"ee9a66e8fa637968d82e9e83d22e2c61a0f2d297af276122f3694772e62d3ec0","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090035242617278464","occurred_at":"2026-08-19T11:16:37Z","signature_id":"0a0c2d83c66feb449a7f4f87715a6640117bc4728f1ca975a82944b2f9039dc5","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090035242617278464","occurred_at":"2026-08-19T11:16:37Z","signature_id":"e5b646e94208fbfa23f1fac70760633b23e949b9bde01b54e28ea0f33aa95610","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"0acfb37bffb4ca551a4e28feb3c3075bb4d22fc8ed7e9fb872066197036c0ac6","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"238018c5ad3f7c198e49aee43504d6edb675df266f6262bd7b5ac202c0734a6e","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"27310541c6ac41b2b15867277516029215eee68b577dd2c8cdd5def731cfa578","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"3a541056af9b027a6109e929bd67604b5073754ff890abd93acfaff50f02560f","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"4761349e386d913979b40fe0b4022048cffc3c3fdf92f8f9e42ba5d0cb06da16","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"508abeacf94e71536edd29c58a77ec4b70b98367a1cad9c7784924a018e77099","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"8e6bebd5be6cbfa0c7c75bcd150afeb11ef0a28af22bd5a0ad8ff061092869e7","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"9ab123f5de31a039a03a1f1e26d3e5d42a79d3aa49f1e86d7896c2263ebbbdb6","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"c8797ce0def20fde7cfb2ece808b9a6049c4d35b238a23cf6bd47744bb8684d5","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089850706507862016","occurred_at":"2026-08-18T23:03:30Z","signature_id":"542bf063b6082a24302d050981d9ec3a40ee2fe9440de9d715588e5249377b06","pr_number":6132,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089850706507862016","occurred_at":"2026-08-18T23:03:30Z","signature_id":"ddf51fc8dcda464e3526756c16d1b87d3323154945573939c68ea3067c1a4c99","pr_number":6132,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089829163719462912","occurred_at":"2026-08-18T21:37:54Z","signature_id":"404b91b9dd8a16071c6fb3245728281b0c55f3979f6f7be12fa2487d75d84592","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089829163719462912","occurred_at":"2026-08-18T21:37:54Z","signature_id":"42d571ee7aca64768a2f8b6c72643ab126c4b8b05f1453363d2baf9938eef40b","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089829163719462912","occurred_at":"2026-08-18T21:37:54Z","signature_id":"b0fd1981a06bd67f7bc5f42996f0a8526669e356d53a52c8215d2dace9177a0c","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089829163719462912","occurred_at":"2026-08-18T21:37:54Z","signature_id":"f3597103b120a9032c74ab487a922a785f46b970f9643d281f3395de402f9925","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089809279165927424","occurred_at":"2026-08-18T20:18:53Z","signature_id":"a18aaa44045e6cb1a7b29b92609caa42a422e1a579558a7cd48c5de3d17c6fc9","pr_number":6562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089796885723222016","occurred_at":"2026-08-18T19:29:38Z","signature_id":"7085c72b738d2b440b85f430f156a733b31a51af5bc55953fcdbe9568293efa3","pr_number":6592,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089768451781431296","occurred_at":"2026-08-18T17:36:39Z","signature_id":"e2d10c3f48bf4e3859687d65b1116a54374fd04e4eb50108f8ddf83f5f396beb","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089759198756737024","occurred_at":"2026-08-18T16:59:53Z","signature_id":"605cfda98c31f08d6c31048cdcef2559abcab4af41205f4af2ca03ee557cac68","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089759198756737024","occurred_at":"2026-08-18T16:59:53Z","signature_id":"6b202030ca324a73d9c91d59cc4bea9b18ddb93f8b2eafd36ec973fba5cdc054","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089759198756737024","occurred_at":"2026-08-18T16:59:53Z","signature_id":"77cc4527ae1e1c0697e1184464b467afe8888e5d421670c83ed66af09f41f38d","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089759198756737024","occurred_at":"2026-08-18T16:59:53Z","signature_id":"9ec6c73054ea99d99bcfed895d3cd5d7d410a1af696fba8958546697e471f0ae","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089759198756737024","occurred_at":"2026-08-18T16:59:53Z","signature_id":"ad04731b4e8b5bd694adce61e23585b04ab5e451765c03aa7a2e7fddc54ab9ef","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089759198756737024","occurred_at":"2026-08-18T16:59:53Z","signature_id":"d1c9c13786922c60039653a23ace044945796e26e377f67aef5fb288b379d1c0","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089757851600818176","occurred_at":"2026-08-18T16:54:32Z","signature_id":"158d1561171b13b72d308cab57b48f4350709dc7070f047859dcb7374f9ad21d","pr_number":5880,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089757851600818176","occurred_at":"2026-08-18T16:54:32Z","signature_id":"7c0d86ea3f20b10aba44c14c82ebce43211518b5ee81f3e0e7b350ea878108f8","pr_number":5880,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089757851600818176","occurred_at":"2026-08-18T16:54:32Z","signature_id":"bfcdcd4ef959f83637f4499dd86450e07606635723adb2bc25a851c172acd1b4","pr_number":5880,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089753182908452864","occurred_at":"2026-08-18T16:35:55Z","signature_id":"1981f8dd89e1df902bc7099d01821bb26e6296ec2ab037c9242d082d35a00c45","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089753182908452864","occurred_at":"2026-08-18T16:35:55Z","signature_id":"29782fcd69b99b4c2ddbd4a212338f7c1e57a736bf3b00545852adcb4e9cdf65","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089753182908452864","occurred_at":"2026-08-18T16:35:55Z","signature_id":"46ad3bfb778237243829bb2839dd081e1f74e18179ff878208f7760c160033c1","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089753182908452864","occurred_at":"2026-08-18T16:35:55Z","signature_id":"a5a6a052c88ec5cba79a926bc16d48a229646e804c9265167d479b369a6cb0f9","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089723779814002688","occurred_at":"2026-08-18T14:39:04Z","signature_id":"148eac94f7584be55efe24b95365051f85fdc9dc483a3e49fd6c95e7f7342026","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089723779814002688","occurred_at":"2026-08-18T14:39:04Z","signature_id":"1a2f65466014d2f638dc2af5a22b3a3a75d5d968a9907c12029c3dce93fa652b","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089713360550498304","occurred_at":"2026-08-18T13:57:44Z","signature_id":"effe80ca17bc2651403c137ba8db85d3ecfdfe1fda98aea3070ca8a23b3f28f0","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708056525934592","occurred_at":"2026-08-18T13:36:10Z","signature_id":"2aa9791653ef0c7dda11a8cc0f38f6443b868051faf9bfc0671d6b4efebe917b","pr_number":6585,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708056525934592","occurred_at":"2026-08-18T13:36:10Z","signature_id":"84a50990ea5cbbe7ffd9231b9f31687181b4b2e6fcbf52647eb0f3cdca1468ce","pr_number":6585,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708056525934592","occurred_at":"2026-08-18T13:36:10Z","signature_id":"98849592a1dd8288d2d06a690d7a97e2a08b80ae4ade2c6ac15c9f226d092adb","pr_number":6585,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708056525934592","occurred_at":"2026-08-18T13:36:10Z","signature_id":"99a14b4958b41ddc0b69120d7654316259d6ccfec96b19bb87d80e5cdc77fa19","pr_number":6585,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708056525934592","occurred_at":"2026-08-18T13:36:10Z","signature_id":"c531c371dd63fcc73cddd67714bcab405dd5e90e4b8204b8ddf23787c8d6cf2d","pr_number":6585,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708056525934592","occurred_at":"2026-08-18T13:36:10Z","signature_id":"f0620a07c3ade76c5cbbf8dd6274108643426b42230d259b204f0b51d78dcae1","pr_number":6585,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6426/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089706907987087360","occurred_at":"2026-08-18T13:32:06Z","signature_id":"02041f3300d04fc73c962749ebd4a66ba4360a42b8e0753e9a822b3dd72b5703","pr_number":6426,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6426/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089706907987087360","occurred_at":"2026-08-18T13:32:06Z","signature_id":"0f46ffd3420531a5c1aec32d09823e8a8a3a35267018de1b781c36e7345dee06","pr_number":6426,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089686738145382400","occurred_at":"2026-08-18T12:11:57Z","signature_id":"5532d4033e88359794c3c2549786f268a2aca86a21040937ae9de796802991d3","pr_number":5880,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089686738145382400","occurred_at":"2026-08-18T12:11:57Z","signature_id":"6388e1d71fb2699b5b3390f0240af44319be7ce812b1a948bf78a5f5e58ba2a6","pr_number":5880,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089665224855523328","occurred_at":"2026-08-18T10:46:28Z","signature_id":"59b6ba9f86bd5023bf571fce831210c1b89c6988049dcb0e6919ac778bcde94e","pr_number":6573,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089665224855523328","occurred_at":"2026-08-18T10:46:28Z","signature_id":"916a45b712204d4b725674c103b0c6f57da03543dc056752d8aa7559f773c6b3","pr_number":6573,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089665224855523328","occurred_at":"2026-08-18T10:46:28Z","signature_id":"dbbd28db37610dfcf1894499e090b980e2b1919d75e457adeb91cec01b4f2b6b","pr_number":6573,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089651610870353920","occurred_at":"2026-08-18T09:52:22Z","signature_id":"29b02f7e35f2e9e7f692e0c72c3aeb3cc8c79f1f77d43c7f59c2b744261a5689","pr_number":6562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089651610870353920","occurred_at":"2026-08-18T09:52:22Z","signature_id":"2f307f048dffdcba9b00827008facc210d1a580598ee70082c02066152c11867","pr_number":6562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089651610870353920","occurred_at":"2026-08-18T09:52:22Z","signature_id":"329deb45a4ba885b21173a08b672a49633e7e40ab463682b3bf4b4823a2dc80a","pr_number":6562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089651610870353920","occurred_at":"2026-08-18T09:52:22Z","signature_id":"d4a867ab8bfc3da23e64e6de85e4e6181d2e2d2727441ec5dd3d04f72c65534d","pr_number":6562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089651610870353920","occurred_at":"2026-08-18T09:52:22Z","signature_id":"dd0b945568e27ade00b155388e2564569317566b9c1ccc7f795e70a0a2a8534c","pr_number":6562,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":101,"runs_affected":101,"failed_runs":101,"run_impact_percent":17.206132879045995,"anchor_occurrences":101,"after_last_push_count":49,"trend_counts":[3,21,23,33,24,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":149,"prior_last_seen_at":"2026-08-21T18:27:18Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":101}],"full_error_samples":["alert fired 3 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-21T19:35:28Z\n  Ended:       2026-08-21T19:44:25Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j9105664-svc\", component=\"frontend\", method=\"get\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/externalauths/{externalauthname}\", severity=\"warning\"\n  Description: The 99th percentile of frontend request latency for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/externalauths/{externalauthname} has exceeded 1 second over the past 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-21T19:36:27Z\n  Ended:       2026-08-21T19:45:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j9105664-svc\", component=\"frontend\", method=\"delete\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The 99th percentile of frontend request latency for delete /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 1 second over the past 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-08-21T19:36:27Z\n  Ended:       2026-08-21T19:43:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j9105664-svc\", component=\"frontend\", method=\"get\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n  Description: The 99th percentile of frontend request latency for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} has exceeded 1 second over the past 5 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-21T19:38:31Z\n  Ended:       2026-08-21T19:46:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j3141504-svc\", component=\"frontend\", method=\"get\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The 99th percentile of frontend request latency for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 1 second over the past 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-21T19:39:30Z\n  Ended:       2026-08-21T19:47:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j3141504-svc\", component=\"frontend\", method=\"get\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n  Description: The 99th percentile of frontend request latency for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters has exceeded 1 second over the past 5 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-21T17:10:50Z\n  Ended:       2026-08-21T17:18:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j1918464-svc\", component=\"frontend\", method=\"put\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The 99th percentile of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 1 second over the past 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-21T17:27:51Z\n  Ended:       2026-08-21T17:33:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j1918464-svc\", component=\"frontend\", method=\"post\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/requestadmincredential\", severity=\"warning\"\n  Description: The 99th percentile of frontend request latency for post /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/requestadmincredential has exceeded 1 second over the past 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6652/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090868362149105664","occurred_at":"2026-08-21T18:27:18Z","signature_id":"9ccad5c6abc73817f266002aafa0e53cefc97f74979f093344241981950c9562","pr_number":6652,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6650/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090866482933141504","occurred_at":"2026-08-21T18:19:50Z","signature_id":"80a92b3760718096cd754f61c5e35a5af2d9b3135a3e9fe48e23b48a27c4d6f7","pr_number":6650,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090837156841918464","occurred_at":"2026-08-21T16:23:18Z","signature_id":"dc9a448fa5fa6e2fcf21e4a673de64ca41bd972130cd2d303b365ed97abaad78","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090833818511478784","occurred_at":"2026-08-21T16:10:02Z","signature_id":"dc9a448fa5fa6e2fcf21e4a673de64ca41bd972130cd2d303b365ed97abaad78","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090833794218070016","occurred_at":"2026-08-21T16:09:56Z","signature_id":"fcec3585b17892e01a8a8d3acce94709e69a2c001f4b79a074973e51b2eafdf9","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090803404556210176","occurred_at":"2026-08-21T14:09:11Z","signature_id":"e0137a945f1e2a54ce81a232b4041838395e6a1684d4fcae4f95e8d894658012","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090803025537929216","occurred_at":"2026-08-21T14:07:41Z","signature_id":"497befd8d2afd386b85a479e07ff0f9765dd27d9ab898de2a689c72a9da46088","pr_number":6646,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6647/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090794933425278976","occurred_at":"2026-08-21T13:35:19Z","signature_id":"58810fa2d3c9ff16a25b6f2f11eecbdb21a8eb7c99871b232f6985ee1705e3ae","pr_number":6647,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090753518062276608","occurred_at":"2026-08-21T10:50:57Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090745162878685184","occurred_at":"2026-08-21T10:17:45Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6640,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090743357851570176","occurred_at":"2026-08-21T10:10:35Z","signature_id":"58810fa2d3c9ff16a25b6f2f11eecbdb21a8eb7c99871b232f6985ee1705e3ae","pr_number":6625,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090725692609138688","occurred_at":"2026-08-21T09:00:23Z","signature_id":"e20b12b46b2b1548a3a16720272c3402c384fc87cdfc8f7364b151aa7395d0a4","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090717616074133504","occurred_at":"2026-08-21T08:28:17Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090717583954153472","occurred_at":"2026-08-21T08:28:10Z","signature_id":"58810fa2d3c9ff16a25b6f2f11eecbdb21a8eb7c99871b232f6985ee1705e3ae","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090683781416488960","occurred_at":"2026-08-21T06:13:51Z","signature_id":"2c875be0e7e1b99f532d6815b3e15d00a56345749d48b8894ad74303128fd231","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090678628760162304","occurred_at":"2026-08-21T05:53:22Z","signature_id":"497befd8d2afd386b85a479e07ff0f9765dd27d9ab898de2a689c72a9da46088","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"19a00a393ae2b995155105e1565d2edf169aa75402e7aa5d511c8fb52a234e24","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090659366838472704","occurred_at":"2026-08-21T04:36:40Z","signature_id":"d2967305299147339686eaf4edf2881f0b317bdd1a1bb179903e51ff8ef771b4","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090649438694412288","occurred_at":"2026-08-21T03:57:23Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090647994339692544","occurred_at":"2026-08-21T03:51:38Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090619074294845440","occurred_at":"2026-08-21T01:56:43Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090612481398411264","occurred_at":"2026-08-21T01:30:28Z","signature_id":"e0137a945f1e2a54ce81a232b4041838395e6a1684d4fcae4f95e8d894658012","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090598725549297664","occurred_at":"2026-08-21T00:35:52Z","signature_id":"ecaa7ce646e97684ae8835ba9c678a9ebeb1e7881117c328a72fd022e3e1949c","pr_number":6630,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090587456947621888","occurred_at":"2026-08-20T23:51:05Z","signature_id":"bb91ddee5e06429218a8675042f0d96a4e566f9e4db42274ff778c8f11f500fd","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090580629858029568","occurred_at":"2026-08-20T23:23:55Z","signature_id":"ffd256ac63b2ff6003a6525bcb93a7edcb7673cf0154ec5c0226a9384ccdeba5","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090577382321688576","occurred_at":"2026-08-20T23:10:58Z","signature_id":"976bbcff91a0b028f2b0d3552b44e9606b7e58c2c2207457e9aaf3cd66c93efd","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090576352783634432","occurred_at":"2026-08-20T23:06:58Z","signature_id":"58810fa2d3c9ff16a25b6f2f11eecbdb21a8eb7c99871b232f6985ee1705e3ae","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090575344808497152","occurred_at":"2026-08-20T23:02:57Z","signature_id":"bb91ddee5e06429218a8675042f0d96a4e566f9e4db42274ff778c8f11f500fd","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090566162734649344","occurred_at":"2026-08-20T22:26:26Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6630,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090555484766474240","occurred_at":"2026-08-20T21:44:02Z","signature_id":"bd2fe2f5d4a8ce19f94859a4e624c3c5667704e60cc19879ec4b40432574a576","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090554196767019008","occurred_at":"2026-08-20T21:38:55Z","signature_id":"58810fa2d3c9ff16a25b6f2f11eecbdb21a8eb7c99871b232f6985ee1705e3ae","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090553390097502208","occurred_at":"2026-08-20T21:35:43Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090549448319963136","occurred_at":"2026-08-20T21:20:03Z","signature_id":"e0137a945f1e2a54ce81a232b4041838395e6a1684d4fcae4f95e8d894658012","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090548196454436864","occurred_at":"2026-08-20T21:15:05Z","signature_id":"ecaa7ce646e97684ae8835ba9c678a9ebeb1e7881117c328a72fd022e3e1949c","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6633/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090523260918697984","occurred_at":"2026-08-20T19:36:00Z","signature_id":"ecaa7ce646e97684ae8835ba9c678a9ebeb1e7881117c328a72fd022e3e1949c","pr_number":6633,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6632/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090522418434019328","occurred_at":"2026-08-20T19:32:39Z","signature_id":"fcec3585b17892e01a8a8d3acce94709e69a2c001f4b79a074973e51b2eafdf9","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514791213830144","occurred_at":"2026-08-20T19:02:20Z","signature_id":"ecaa7ce646e97684ae8835ba9c678a9ebeb1e7881117c328a72fd022e3e1949c","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"d2967305299147339686eaf4edf2881f0b317bdd1a1bb179903e51ff8ef771b4","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090501897923334144","occurred_at":"2026-08-20T18:10:59Z","signature_id":"ecaa7ce646e97684ae8835ba9c678a9ebeb1e7881117c328a72fd022e3e1949c","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090489518846119936","occurred_at":"2026-08-20T17:21:54Z","signature_id":"58810fa2d3c9ff16a25b6f2f11eecbdb21a8eb7c99871b232f6985ee1705e3ae","pr_number":6278,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090484744067223552","occurred_at":"2026-08-20T17:02:48Z","signature_id":"ecaa7ce646e97684ae8835ba9c678a9ebeb1e7881117c328a72fd022e3e1949c","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6625/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090479903731355648","occurred_at":"2026-08-20T16:43:42Z","signature_id":"ecaa7ce646e97684ae8835ba9c678a9ebeb1e7881117c328a72fd022e3e1949c","pr_number":6625,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090463536332410880","occurred_at":"2026-08-20T15:09:03Z","signature_id":"41651525e2aba85d110545005660d80238d17590a754aa3d1b78632adaaaf0ed","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090448552185565184","occurred_at":"2026-08-20T14:37:48Z","signature_id":"b7fbd103a195b66ac94bc7078697b03cb8d35c8e0fee824888587e1e72a89148","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090394899235999744","occurred_at":"2026-08-20T11:05:56Z","signature_id":"bb91ddee5e06429218a8675042f0d96a4e566f9e4db42274ff778c8f11f500fd","pr_number":6592,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090373033880457216","occurred_at":"2026-08-20T09:39:03Z","signature_id":"4fcebcb05bfc80f2d35d69227639a0e21e16fdc0d0a4461c0b08ead5dba75150","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090355263239360512","occurred_at":"2026-08-20T08:28:26Z","signature_id":"955efdfa565c09addf9d705f480891c18ea8c9dcafa7daf71cdc742a8e04bd87","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"de88c780ac57910f5dd0f3ea4c4fe8777e1ae26bc3f67f21242d59fc868296e7","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090344759288664064","occurred_at":"2026-08-20T07:46:41Z","signature_id":"d2967305299147339686eaf4edf2881f0b317bdd1a1bb179903e51ff8ef771b4","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090334446552092672","occurred_at":"2026-08-20T07:05:43Z","signature_id":"a9ecbd925234d950c6ec9a4c432a806852925b879c184fba9cd8c97ceceb05f5","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6616/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090329417594179584","occurred_at":"2026-08-20T06:45:44Z","signature_id":"e0137a945f1e2a54ce81a232b4041838395e6a1684d4fcae4f95e8d894658012","pr_number":6616,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090283973790928896","occurred_at":"2026-08-20T03:45:09Z","signature_id":"b998d1848e9e74b3cafa33bfa1cafbba26000bfe45b96ebc3b692b721613768d","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090266089802960896","occurred_at":"2026-08-20T02:34:05Z","signature_id":"bb91ddee5e06429218a8675042f0d96a4e566f9e4db42274ff778c8f11f500fd","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090253315500675072","occurred_at":"2026-08-20T01:43:20Z","signature_id":"58810fa2d3c9ff16a25b6f2f11eecbdb21a8eb7c99871b232f6985ee1705e3ae","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"521837880a2f6f77d1ffd5ab821dd3785b6f84bb96c295368a3577aaaf98e844","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090214858837463040","occurred_at":"2026-08-19T23:10:31Z","signature_id":"ecaa7ce646e97684ae8835ba9c678a9ebeb1e7881117c328a72fd022e3e1949c","pr_number":6614,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090212568353214464","occurred_at":"2026-08-19T23:01:14Z","signature_id":"e0137a945f1e2a54ce81a232b4041838395e6a1684d4fcae4f95e8d894658012","pr_number":6588,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090212556621746176","occurred_at":"2026-08-19T23:01:09Z","signature_id":"58810fa2d3c9ff16a25b6f2f11eecbdb21a8eb7c99871b232f6985ee1705e3ae","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090206132864815104","occurred_at":"2026-08-19T22:35:50Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090198433062195200","occurred_at":"2026-08-19T22:05:11Z","signature_id":"f43d4b8968a344f7689d525224fffe656bfdbfeb15dfa483f233040306cfd422","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090192149080444928","occurred_at":"2026-08-19T21:40:16Z","signature_id":"c17108aafdd26274dff3fa12598513024038c96e6670180dc72078501ecefea5","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6572/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090182266104844288","occurred_at":"2026-08-19T21:01:00Z","signature_id":"bb91ddee5e06429218a8675042f0d96a4e566f9e4db42274ff778c8f11f500fd","pr_number":6572,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090179751976112128","occurred_at":"2026-08-19T20:51:01Z","signature_id":"e0026eefcc9d021330722912e2d8eb29e1d3fde3e81032401b2fd763208297bd","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090166116277030912","occurred_at":"2026-08-19T19:56:50Z","signature_id":"58810fa2d3c9ff16a25b6f2f11eecbdb21a8eb7c99871b232f6985ee1705e3ae","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090159046848942080","occurred_at":"2026-08-19T19:28:44Z","signature_id":"bb91ddee5e06429218a8675042f0d96a4e566f9e4db42274ff778c8f11f500fd","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090119905717260288","occurred_at":"2026-08-19T16:53:12Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090119424534122496","occurred_at":"2026-08-19T16:51:17Z","signature_id":"ac7c23c664839f4ba518b63f650cf0315a11f7d86a647ed599ba5ba0a729f061","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090105093117448192","occurred_at":"2026-08-19T15:54:21Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6592,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6426/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090100918157578240","occurred_at":"2026-08-19T15:37:04Z","signature_id":"ecaa7ce646e97684ae8835ba9c678a9ebeb1e7881117c328a72fd022e3e1949c","pr_number":6426,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090085989094526976","occurred_at":"2026-08-19T14:38:26Z","signature_id":"fcec3585b17892e01a8a8d3acce94709e69a2c001f4b79a074973e51b2eafdf9","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6600/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090085087973478400","occurred_at":"2026-08-19T14:34:51Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6600,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090055453999370240","occurred_at":"2026-08-19T12:37:06Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090035242617278464","occurred_at":"2026-08-19T11:16:37Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089998936298229760","occurred_at":"2026-08-19T08:52:31Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6599,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089982500192391168","occurred_at":"2026-08-19T07:47:12Z","signature_id":"a179f310e31feef8931f82d5879b3d20bc9a6c8f6cae34f97512a9f6c979cd54","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089907805891858432","occurred_at":"2026-08-19T02:50:24Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6558/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089876846182141952","occurred_at":"2026-08-19T00:47:22Z","signature_id":"dec22c8d7025ad850ec3e80affbb82b1b1ebdec8a7deb9b908c0fcd23d44b728","pr_number":6558,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6593/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089864988096204800","occurred_at":"2026-08-19T00:00:06Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6593,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089843361052626944","occurred_at":"2026-08-18T22:34:19Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6558/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089839143172444160","occurred_at":"2026-08-18T22:17:33Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6558,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089829163719462912","occurred_at":"2026-08-18T21:37:54Z","signature_id":"ecaa7ce646e97684ae8835ba9c678a9ebeb1e7881117c328a72fd022e3e1949c","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6593/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089824392346341376","occurred_at":"2026-08-18T21:18:52Z","signature_id":"f084efd97dc1b5e14a6ace8ccc7148d5b707f4473c2e32a14b987a14e5a4ca12","pr_number":6593,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089820335682621440","occurred_at":"2026-08-18T21:02:49Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6558/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089788762895683584","occurred_at":"2026-08-18T18:57:22Z","signature_id":"7b129d827139a0c5c98cd4bd8c06fd567a21bf618255f2196fab01a3481c4365","pr_number":6558,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089786820278620160","occurred_at":"2026-08-18T18:48:55Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6576/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089786196182962176","occurred_at":"2026-08-18T18:47:05Z","signature_id":"ce3c32a6da1132d17285ee9cbf686dc2a49a23b3e2bfb4ec750f1ad920901cc5","pr_number":6576,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089759198756737024","occurred_at":"2026-08-18T16:59:53Z","signature_id":"4fcebcb05bfc80f2d35d69227639a0e21e16fdc0d0a4461c0b08ead5dba75150","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089753182908452864","occurred_at":"2026-08-18T16:35:55Z","signature_id":"7caa05483a66ff952c3422b319bb98b5d3593a3bf381104824f8c314ebc91c52","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089749669939777536","occurred_at":"2026-08-18T16:22:01Z","signature_id":"778bb57f26f57718a102016788d1926ef865383c03334952b8595b59b2916eff","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089740517767122944","occurred_at":"2026-08-18T15:45:39Z","signature_id":"fad5d35a11971136c7ab041e4e6d86f0c5c45813de78c781c35e62a0b335b07d","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089722221688786944","occurred_at":"2026-08-18T14:32:57Z","signature_id":"311566099b788fc3964310d54e3d0ee55a815e4049362f4831fd8a8a73f9bc9f","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089714962824957952","occurred_at":"2026-08-18T14:04:06Z","signature_id":"58810fa2d3c9ff16a25b6f2f11eecbdb21a8eb7c99871b232f6985ee1705e3ae","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708056525934592","occurred_at":"2026-08-18T13:36:10Z","signature_id":"311566099b788fc3964310d54e3d0ee55a815e4049362f4831fd8a8a73f9bc9f","pr_number":6585,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6564/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089691879867158528","occurred_at":"2026-08-18T12:32:19Z","signature_id":"ce3c32a6da1132d17285ee9cbf686dc2a49a23b3e2bfb4ec750f1ad920901cc5","pr_number":6564,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089679732839813120","occurred_at":"2026-08-18T11:44:07Z","signature_id":"d8e5e45bb1905c9be7d3992da9e83bef866a4306f82084f3f56e91cadc3606c7","pr_number":6553,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089674709242220544","occurred_at":"2026-08-18T11:24:09Z","signature_id":"497befd8d2afd386b85a479e07ff0f9765dd27d9ab898de2a689c72a9da46088","pr_number":6564,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089651610870353920","occurred_at":"2026-08-18T09:52:22Z","signature_id":"81a1abf69729f913c8baed9928bf7aec208df11f9bd62fdf3f0cc99492ffa32a","pr_number":6562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089604166161797120","occurred_at":"2026-08-18T06:43:50Z","signature_id":"f963377877adce2b702f32a07f68ea30e6b30f0fb833ff7112e55d79ec6f1840","pr_number":6553,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089600967690424320","occurred_at":"2026-08-18T06:31:08Z","signature_id":"81a55cd76ef01be3a71d08631473f8a619f3706a3c3e81c9bfe4c3fc003e584a","pr_number":6566,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":70,"runs_affected":70,"failed_runs":70,"run_impact_percent":11.925042589437819,"anchor_occurrences":70,"after_last_push_count":16,"also_in":["INT","PROD","STG"],"trend_counts":[2,0,0,0,70,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-17"],"prior_runs_affected":73,"prior_last_seen_at":"2026-08-21T21:06:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":22},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":48}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:194]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00195b230\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21, targetMinor=4.22) failed: clusterversion status.history has no version in target minor \"4.22\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:194]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00195b230\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21, targetMinor=4.22) failed: clusterversion status.history has no version in target minor \"4.22\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:194]: Timed out after 2700.006s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001282120\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21, targetMinor=4.22) failed: clusterversion status.history has no version in target minor \"4.22\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:194]: Timed out after 2700.006s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001282120\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21, targetMinor=4.22) failed: clusterversion status.history has no version in target minor \"4.22\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.005s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0001dcbe8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21, targetMinor=4.22) failed: clusterversion status.history has no version in target minor \"4.22\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.005s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0001dcbe8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21, targetMinor=4.22) failed: clusterversion status.history has no version in target minor \"4.22\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090908436404375552","occurred_at":"2026-08-21T21:06:33Z","signature_id":"4090f06022a82484d544da5fc0f946bd246078a58a2fc784c8020ca944fe069f","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894768476262400","occurred_at":"2026-08-21T20:12:08Z","signature_id":"0382ce8a1155781d97bef6495e5c7a6669d7186d4d84f3bc2bf06b0a0d015de3","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894762604236800","occurred_at":"2026-08-21T20:12:07Z","signature_id":"ab796cc2f837d109fd6c7397a4be71f27775f755efc3f9b1ad40beb70f93595f","pr_number":6646,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894756736405504","occurred_at":"2026-08-21T20:12:05Z","signature_id":"2c1c634c357c5606e87b00bc23ab15eefacde5af4a249ffc1395fa0077cb9efc","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894675127832576","occurred_at":"2026-08-21T20:11:52Z","signature_id":"ccc09dd771e181c0dae77e9a7fd57d84a7a07fd1b75591cad8507db7ecb4c61f","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090884661910704128","occurred_at":"2026-08-21T19:32:04Z","signature_id":"ab0c49f3bf28254b3148cedfad87253207cd57a59f17c10afbf49b05b0290a36","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090882234021056512","occurred_at":"2026-08-21T19:22:25Z","signature_id":"922650986953b8202aefb7a4964bc0321d02b0ab4447ecdf89a383072d8c5f5b","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6593/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090879591731171328","occurred_at":"2026-08-21T19:11:50Z","signature_id":"2ddc98ca1c293c2673fbf12ebd368e974d9e10640b768c7a482963ef32c95ae1","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090878713473273856","occurred_at":"2026-08-21T19:08:16Z","signature_id":"4c4fa7fb1bf625b2d3247b2379b358428d0bce0e17e28d67498458083669e75b","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090877968120287232","occurred_at":"2026-08-21T19:04:55Z","signature_id":"bda765f4ccf539c4ee0129f74962705e6379bbba6aa02e9859330a3d2c5dc3e6","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090871776832655360","occurred_at":"2026-08-21T18:40:52Z","signature_id":"b1670c7da43304dc559d677b292763c03c30dda0db3d1248632ea3056d150ddb","pr_number":6651,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6650/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090866482933141504","occurred_at":"2026-08-21T18:19:50Z","signature_id":"19dafc7bc774815638556ca7b37fef1969115a880a613ecf7527e2301d85187a","pr_number":6650,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090849047588900864","occurred_at":"2026-08-21T17:10:33Z","signature_id":"b8688968de3c5ef5b2db6bfa888b16997a3b71cf9d9183396c6c376709df7cc7","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090849031038177280","occurred_at":"2026-08-21T17:10:29Z","signature_id":"ffece2232b540a63d0803ff30b0e3c8fbef0ab71e70b9d4c20af731d95bedc34","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090849020749549568","occurred_at":"2026-08-21T17:10:27Z","signature_id":"c72c6838ad7c4bae62f8f4cd0430795160801a2135e286abad7a5f87c488245c","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090848752863547392","occurred_at":"2026-08-21T17:09:23Z","signature_id":"e823c669604f208a00298d07aa6dc273511165907708d46e387b703b047b5cb9","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090837373561606144","occurred_at":"2026-08-21T16:23:45Z","signature_id":"397a410f407a5210d3f56529fec7e6e07d0b098af14ba62c19445a400fae3187","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090837156841918464","occurred_at":"2026-08-21T16:23:18Z","signature_id":"1ef873971e878e4763e4ff1868eab81d4b41b24afe427ec15e91f0aa15eaacf7","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6647/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090836865526534144","occurred_at":"2026-08-21T16:22:00Z","signature_id":"b6aababddc1cea32acce54ad7f303b1d7e1bb032433f83a7ef1bab88320054a1","pr_number":6647,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090836156110344192","occurred_at":"2026-08-21T16:19:20Z","signature_id":"a3c5ac894514a4d5681ed7a6c47cdc78a663701ede7adcf20cd9e767c2555485","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090833818511478784","occurred_at":"2026-08-21T16:10:02Z","signature_id":"f41376e0fd047969cb02e607037104a05ba22266b1beaf1cb69eafa2b436d7a6","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090833794218070016","occurred_at":"2026-08-21T16:09:56Z","signature_id":"c95408ad8246dd7163fe031ec6185530c3613df10726090cedfabb8d55ae263d","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090817798367350784","occurred_at":"2026-08-21T15:06:21Z","signature_id":"148240e213c49694b88bf4e5f3bc8b75f8e2f1924a548767dfb616662fd80005","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090817534927310848","occurred_at":"2026-08-21T15:05:19Z","signature_id":"b3d8c5ac4a8dead5518d320568d761e00060b54e4d32594b44b568f24b398b97","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090811015728992256","occurred_at":"2026-08-21T14:31:41Z","signature_id":"a5825696e84694b99c030c756284c4e9a073e005bd400906c8cc01050aa12a6f","pr_number":6621,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"0ecd479311dc3e3983f7b5d80071fce53a706e25650eb18d08bdde3f73141495","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090805741328797696","occurred_at":"2026-08-21T14:18:28Z","signature_id":"9faef49dd63ed61c994ede57fd7dfae2df4d6be0db3c901d202045d895b077c3","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090804889495015424","occurred_at":"2026-08-21T14:15:05Z","signature_id":"bccfa8e61fa208ededbeb52840eb11c0c377b390c03f6641d031e037b5cd077f","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090803404556210176","occurred_at":"2026-08-21T14:09:11Z","signature_id":"135e19c4ac0830ce6714eb18154f43c293f34ec3cc7d98b682735da5fbd37f49","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090803249215967232","occurred_at":"2026-08-21T14:08:34Z","signature_id":"9da15eb1a64b2d2af52bf57fad31504e43850bec545a73e48fc7ccb8884a3f81","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090803211123298304","occurred_at":"2026-08-21T14:08:25Z","signature_id":"42e637ddf5c18937fb7b0f3287feddb3a95816fa6dfdc6a1a0b2c83598393fe2","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090803025537929216","occurred_at":"2026-08-21T14:07:41Z","signature_id":"a0632bb2f2060849353ff56a6005a99e05ef66ab605602be26920a61645415d2","pr_number":6646,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6647/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090794933425278976","occurred_at":"2026-08-21T13:35:19Z","signature_id":"75a337ac90d8611397e2787f50022df937374eb5fa76f2ca3d1f8c06fbd055a5","pr_number":6647,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6643/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090790935381151744","occurred_at":"2026-08-21T13:19:38Z","signature_id":"904aeee3492aac19dd9b67e842266a7947c6c5c1d8021b2c491d6d1badfbfbcd","pr_number":6643,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090784604393836544","occurred_at":"2026-08-21T12:54:29Z","signature_id":"060103ebf791bdc252e9fd7c3f2004368fc7294240ee45e80feaf2339b2c6c87","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090780869554868224","occurred_at":"2026-08-21T12:39:38Z","signature_id":"73a8bfe1e11251ff2d14478f901cefed6869821c0b0b81674cf8b8c4650624b6","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090779685267968000","occurred_at":"2026-08-21T12:34:56Z","signature_id":"812d71e6470288c16de9c3ff3c3d3de5bf15d2ed32e2e24c2f8078f4e7070425","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090779285378830336","occurred_at":"2026-08-21T12:33:21Z","signature_id":"96bbb2ae16d7c3fed47071662d0ea92d4f245f605dc4b6953361f7a32c5237e9","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090776820973572096","occurred_at":"2026-08-21T12:23:15Z","signature_id":"8e684d65c1dc3791016178d8e12d6c74b1706b047b2be7de1090ab37a3d8a6d8","pr_number":6640,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090773034322366464","occurred_at":"2026-08-21T12:08:30Z","signature_id":"7f2d125c3d23463139685011d35022cc1cc318fb866357defe713bd166e72735","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090770393882169344","occurred_at":"2026-08-21T11:58:01Z","signature_id":"eb1ffbf143a86bb0416996b0887acda8192476da3ccf504edbb9c3becda710b5","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090769277882732544","occurred_at":"2026-08-21T11:53:35Z","signature_id":"f9d437562fc5f93c2546a0272f5fefd7436a69a1b41db0828c34e8c779ed308a","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090768490683174912","occurred_at":"2026-08-21T11:50:27Z","signature_id":"f7c024e4f79c41c3ae7a7e53f4c95498df76592b592dfd6b8a171ee3536433db","pr_number":6646,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"8decc7f4d214fff8194d1174e836c4092ecd3b32e8d9bf66efc240fc275fd185","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090766839033368576","occurred_at":"2026-08-21T11:43:53Z","signature_id":"abf07e40a05be4b14049a34485a64c8a795ac2ad5eb7c759ffd4d80c9874faf5","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6643/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090757087574364160","occurred_at":"2026-08-21T11:05:08Z","signature_id":"d669bfaa1670fbad664b5ac73b7c79489bae68126c49cf2f1fba0eabc1c51316","pr_number":6643,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090753518062276608","occurred_at":"2026-08-21T10:50:57Z","signature_id":"6bfa7ec24001ab7354b3bd27d9ff458ec36b57a265693ecaec9a2df3f646d744","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090751316451135488","occurred_at":"2026-08-21T10:42:12Z","signature_id":"6dd2fb179ec22ce2728df7bd94aa1cce6d1a9cef31014de181ba8e933b51ba65","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090739392959221760","occurred_at":"2026-08-21T09:54:49Z","signature_id":"0a8114641b5c7afff27980ca306ad679af93fd5d4751a1dac3d983b42b867fda","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090725937179004928","occurred_at":"2026-08-21T09:00:54Z","signature_id":"59425a1054fbda560f28ca3e279dd01b90478c1e2854741e205910c434c21114","pr_number":6625,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090725692609138688","occurred_at":"2026-08-21T09:00:23Z","signature_id":"2141037c63ca4477ad055f5b74bf51e5e0c128c55d5a4ca3f343afe64247d3d9","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090720049605120000","occurred_at":"2026-08-21T08:37:58Z","signature_id":"a1d1494c1cd89e3934b69d63bf00641bcb31a00c1a245a6e4b92280f32d1b0c4","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090717616074133504","occurred_at":"2026-08-21T08:28:17Z","signature_id":"bd8a8f86b00d1c8cfb9c1cdaae4055ffba2dc8890c177c72364ba0ee601d38d2","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090717583954153472","occurred_at":"2026-08-21T08:28:10Z","signature_id":"681bf3b2806e864c549cf7fe8c2a3906ebe0676693b2b19614ceeb29b9cae8f9","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090711731759222784","occurred_at":"2026-08-21T08:04:32Z","signature_id":"16b25b47d2325fbbc2d1941dc8824c4a80b04e5f6973b34ca6ffea24a1207acc","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090693922316619776","occurred_at":"2026-08-21T06:54:08Z","signature_id":"64e25d112e0695e1d034cd57b56d23c9102d4d7807f92050e1757b8779c09444","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090688754837622784","occurred_at":"2026-08-21T06:33:36Z","signature_id":"0a94ed3abfcb10826ec265b69eee6c3d3f177e2b400b849a60c6c232870d25b2","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090683781416488960","occurred_at":"2026-08-21T06:13:51Z","signature_id":"3f8187d860ff81ce8e842c107134d560cc17b74b0d9a3d6f7bc6c288b56ff143","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090678628760162304","occurred_at":"2026-08-21T05:53:22Z","signature_id":"cd5e3a93de1c1b4a0f571db23612917f911fead22146e5cdddc4af2900037c22","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090677551943913472","occurred_at":"2026-08-21T05:49:05Z","signature_id":"b33468305bac6091a451daaf803ba20e051b55f2d05ad65d539c7a79fec82cdb","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090676319984553984","occurred_at":"2026-08-21T05:44:12Z","signature_id":"275c9b1cc9ab11aaa390a42bdd290c7343b52822e3c9e5fc88e7b2329f85da3e","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090660118717796352","occurred_at":"2026-08-21T04:39:49Z","signature_id":"de77568282fe39219f9c9d230c95485c7c2a386ecba55052652842309ae74587","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090659366838472704","occurred_at":"2026-08-21T04:36:40Z","signature_id":"17d1501457e51a3d2cc1966853d30f056c3c01a9b8dfa3fc989a9476679ca64c","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657643948412928","occurred_at":"2026-08-21T04:29:59Z","signature_id":"ea6bd51c7d5e9d4edc59703a240cffeabbf9e75056df123880d04b0e8d5aa06a","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"2317d1519d946e5b068390bd95b67193bef53c958391cf5cfe1e06907335eb51","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090649438694412288","occurred_at":"2026-08-21T03:57:23Z","signature_id":"c4ec8921333fd1436f1e90888ca42212a45ef348fe432fce088bc6c75bb586b8","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090647994339692544","occurred_at":"2026-08-21T03:51:38Z","signature_id":"a89bb15d917c56506b65981e3e85aca7499e44b9e24eaaac137e1fcc0c0b4f38","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090646203308969984","occurred_at":"2026-08-21T03:44:31Z","signature_id":"d1e1dcdaf209cb4d50df4bb2b230ba25fd816f993dd504a0b70c2a90b8ac9196","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090644128227397632","occurred_at":"2026-08-21T03:36:17Z","signature_id":"2f7014bea9a6fd8030e9f98db125d9ce51d2f0c53cd554d31900751eb41c884b","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090643626764800000","occurred_at":"2026-08-21T03:34:17Z","signature_id":"241ca05585b9c433095e06f7ae32c935c81bc64b5e7402465f48ef00f091cf14","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":65,"runs_affected":65,"failed_runs":65,"run_impact_percent":11.073253833049403,"anchor_occurrences":65,"after_last_push_count":70,"trend_counts":[9,14,9,14,12,2,0,14],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":348,"prior_last_seen_at":"2026-08-22T05:45:38Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":65}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-25T00:35:25Z\n  Ended:       2026-08-25T00:56:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1843200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd9551d19b6bblvufm3l3a421htavmi-a4k0z5e8x0e3t0f\", pod=\"router-597d99bc98-tlmg9\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sd9551d19b6bblvufm3l3a421htavmi-a4k0z5e8x0e3t0f/router-597d99bc98-tlmg9 has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T23:47:43Z\n  Ended:       2026-08-25T00:09:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4709760-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd8f93h71j1qrcah2q03eja41gojmif-i6w1a1c6f6m8f7l\", pod=\"router-7f97cf9c5-phjp6\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sd8f93h71j1qrcah2q03eja41gojmif-i6w1a1c6f6m8f7l/router-7f97cf9c5-phjp6 has been in a non-ready state for longer than 5 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T22:04:58Z\n  Ended:       2026-08-24T22:16:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9409536-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd6tku01d4e5c0s0k2ocfdtf91cpr8l-d3q6r6q7q0m9f7w\", pod=\"router-57846fcf5b-n5mlw\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sd6tku01d4e5c0s0k2ocfdtf91cpr8l-d3q6r6q7q0m9f7w/router-57846fcf5b-n5mlw has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T22:04:58Z\n  Ended:       2026-08-24T22:16:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9409536-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sd6td57g5fetk5hnhg543eg0jfi9lrt-j5s2f3c0f5x7m8z\", pod=\"router-7586ffccfd-4mvlg\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2sd6td57g5fetk5hnhg543eg0jfi9lrt-j5s2f3c0f5x7m8z/router-7586ffccfd-4mvlg has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092034671041843200","occurred_at":"2026-08-24T23:41:48Z","signature_id":"2f1d9b2b8857e75d809a23da90c2e32bd0a70345c25f6862fc7292ac5c796db3","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092020242174709760","occurred_at":"2026-08-24T22:44:28Z","signature_id":"f8ca631bf4ba3de6a823751843028c53bf8f63552f781736fc0f06124e42925e","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"762c894801e83a205ad61ad7c1a1171fcbbd7db65d82ac38613bccfd72e8ae0d","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"5a62ef9a7062e35c1e10c6c46839368f20045e53d18429e0c93ad1b8118f4ed4","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","signature_id":"05c28635da15b6abb7fa52304cc82cedf7070115a67070cb7058c7bc17ceed55","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091965378614792192","occurred_at":"2026-08-24T19:06:19Z","signature_id":"d9b2b2f48eba30a813d1493c2cf9d779d74129e8e68690c29d48df3b77d5680a","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","signature_id":"84936854ba2393da3d6d4a90cbe465eac2bfa99410eb5ae262e1869d118d0d44","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091939168878858240","occurred_at":"2026-08-24T17:22:18Z","signature_id":"d6263c32bd22604b284f6288625ad3fcd30c678787956b94fe5bf7e9a8e0f6cc","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"ddb17f85d08dfa0f0d60af8e146b060c3f080bb3cb873c1fd946241da0c96c74","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091920407983558656","occurred_at":"2026-08-24T16:06:36Z","signature_id":"c52899aff7bf73e0f12d284cdf7344825014b3fd387fb2f270a8871271bc7acb","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091899442289446912","occurred_at":"2026-08-24T14:44:27Z","signature_id":"32b177871d4669aa26bebdb303a13d4eeafa32be58fe0c29eecd2b7355b315ff","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","signature_id":"d1d5a7d0ff60a9161bf6b8e8b3cd897227641812139ecab6ded5ff874a074bf7","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"41f9fa028fb65313589b4f31abf40b7c37b7c57dd66b9ac548b1148f02514739","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","signature_id":"d55af07fa2830b9ad7b195c673e4e375ec2e39f3e870f6e6e4139ca2af8abfa0","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6593/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091039068275085312","occurred_at":"2026-08-22T05:45:38Z","signature_id":"95aa3aca64e7cc46c2a08e24bb4eb93743dac610ac55bf5e95aee1631ffe4b61","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"b7be27a9228028688d416758c9450df4007cbf195a10011266f7041548c3fec5","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"14ed32dc0216996b1a8878b8a835543278abe81456380f713b5041f8fc293420","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090908436404375552","occurred_at":"2026-08-21T21:06:33Z","signature_id":"ec64a710749d633cd2c44431c9388abb2b8806231c9e0acb1122140d519a61e1","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090882234021056512","occurred_at":"2026-08-21T19:22:25Z","signature_id":"88b630622168ed4766114eb7b40abc17cff2f5a5bfa775cd4914781d0114c005","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6593/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090879591731171328","occurred_at":"2026-08-21T19:11:50Z","signature_id":"28a74d91f38fce537ecd7876c2fbeb1def25cdc046d90f155aa117593c40183d","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6652/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090868362149105664","occurred_at":"2026-08-21T18:27:18Z","signature_id":"cb9e738da98477afa6434f301921aa71901c190c0fa2ae1f4fdd3dc32d9b1f5b","pr_number":6652,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6650/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090866482933141504","occurred_at":"2026-08-21T18:19:50Z","signature_id":"059468d2108c80b7a8e95125cdb23fe8c965fe7682f802b5eaf7c270a982de71","pr_number":6650,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090836156110344192","occurred_at":"2026-08-21T16:19:20Z","signature_id":"1ee6bcd10080625d968e07eac093a0e8c09f764a459c0d9ed1e90b69d0f7986b","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"a05ad038bf9aac70e32cc0368923a71fd08a6b2128f0cdc785f1671acd4c9134","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090753518062276608","occurred_at":"2026-08-21T10:50:57Z","signature_id":"d637ee2c2450b4d6a5abdfb0894110693323297ee0ace44237471e7cf0e1d59a","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"d1fa73718ab2519b4a0e1216040040fd07bcc0f880baf692f5762be6551c9a56","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657643948412928","occurred_at":"2026-08-21T04:29:59Z","signature_id":"ed86faacee79b1d1d24720026b790d34d8418a0d8388cdee3111523a76e60e24","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"59e480ebe494d162afecad172ddf865743624a68f83750f3f473d533160b3b19","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090577382321688576","occurred_at":"2026-08-20T23:10:58Z","signature_id":"f3708a10e3f19aa3eaca1794571c4aaec444f1598c47dd25001400f3078b89a8","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090576104745078784","occurred_at":"2026-08-20T23:05:59Z","signature_id":"6fb8e1ab71971b31e568d57640f39efae16402260414e8b3aa699cc7b204bdf9","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"3468b8c2c816f1903b03c85d762cf81679135c586b96ea19bc866ef680ab082d","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090549448319963136","occurred_at":"2026-08-20T21:20:03Z","signature_id":"ffeecb88b31cc411dc8045e435c12c5f29ddf0da822c623c2e41b8ba06d17b7b","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514755771961344","occurred_at":"2026-08-20T19:02:12Z","signature_id":"06dd656267c6d34d26f117c02a8ccbe440b9f064addf99cfd5b363c42f885b3d","pr_number":6621,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"9d20cffd8d432102b61dbbd4e1ce50696352d2e43889ac63d6df80980c8d111d","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090489518846119936","occurred_at":"2026-08-20T17:21:54Z","signature_id":"c7ca06524fabe9e64053c271f8f0f6a6a5830b6427b478c52145ffa5f10b5b60","pr_number":6278,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090419688172425216","occurred_at":"2026-08-20T12:44:10Z","signature_id":"b7647a84afd55a9c67574cc6ec4034d30cef22f2363e4eff8c48a90300df4629","pr_number":6617,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090389808017838080","occurred_at":"2026-08-20T10:45:42Z","signature_id":"8c462b89c57a050fa84108ca0066a350520f68f99013cdd5ab3d20bf880b0222","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6616/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090329417594179584","occurred_at":"2026-08-20T06:45:44Z","signature_id":"24a02abde95219099d1524e2e5aa7008b06f1e9672a9fe6c323a562037eb7f62","pr_number":6616,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090313734160388096","occurred_at":"2026-08-20T05:43:24Z","signature_id":"f2489c96523fd094164119ff2a1d32086287ac39d481d98edb42a30691de61f6","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090249792243372032","occurred_at":"2026-08-20T01:29:20Z","signature_id":"e0ceb031e603349f821d9e164278112ab870c2f3a5b4a2121c737e4d31ff2768","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"42dbbb683d20ebb408cfab3ddf9a96496e42a39b8db6630d20713d16fa7fec0a","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6555/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090233576732758016","occurred_at":"2026-08-20T00:24:53Z","signature_id":"bf238d0ad10d36467eefe0d07977bf4d6a9dd12b68f3e414cb869ce3d72349cc","pr_number":6555,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090221977028530176","occurred_at":"2026-08-19T23:38:48Z","signature_id":"f1fdf1ff2fb718d1c8a9ff25e58ecafe17e2abf2d47632186ffc24c15fb3ebfd","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090191599463043072","occurred_at":"2026-08-19T21:38:05Z","signature_id":"01966e52482dd9a1375c8351a3593e5fad146e637a4c9387fd4a038652d5eeac","pr_number":6608,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090166116277030912","occurred_at":"2026-08-19T19:56:50Z","signature_id":"d8fd3c206c23d36dd54b4c1a854933e286dfbc3d5cfddcfad693ea3995863e32","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"d72e24ff9b0c8a737023df49a3efa6985c6c8132f311c961219684a83d87232a","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"172fe634a5279b7fca05fd5f9589d216e428a0b4959f4ad85d7e59d71aa0ee3c","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090085264507539456","occurred_at":"2026-08-19T14:35:33Z","signature_id":"611225ad9ec7a09f32026a1d1a439e7cdeea3d31b702dbb4b07bf74fdf162705","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"a0a91a26492bf825ff807684676ca7b8921438ad4fa4db49f7fa7c17c3799856","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6606/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090076575402823680","occurred_at":"2026-08-19T14:00:40Z","signature_id":"03e27710d2fff2dd7d16273a4334ebe0c89597d01d5530e00cb0fd71e159e501","pr_number":6606,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6600/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090054287706034176","occurred_at":"2026-08-19T12:32:25Z","signature_id":"e5a51d0b1db4e0390900606b4a4496ba6e219f60f9b9127ac25cadd7f467125f","pr_number":6600,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089829163719462912","occurred_at":"2026-08-18T21:37:54Z","signature_id":"552d2a90faa33d3842c3588f0d98ddd1249421105941f616914c66fbf3345eb7","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089810122246197248","occurred_at":"2026-08-18T20:22:14Z","signature_id":"66a24c5356ba2f3c0fd45891557f246e0df93bb54ce9535a0c5989b2d1a037dd","pr_number":6562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089759198756737024","occurred_at":"2026-08-18T16:59:53Z","signature_id":"d19b66ea4004d52dc65957d036ea95d999009175d7a91d0acfedf05a2081a703","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089740517767122944","occurred_at":"2026-08-18T15:45:39Z","signature_id":"6f6c8f248c065a1ff07567c37f21290822cfbaf4f2301f2816905b4f3b84e3fd","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"4f776567a937d9b683887f57e28c5480e0eb59800c60a205ce7433cc4e8785c8","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089723779814002688","occurred_at":"2026-08-18T14:39:04Z","signature_id":"d4c0a344517decc66a540bf70d542ac1f5a5364440e546c11c84ba68f75cb147","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089713360550498304","occurred_at":"2026-08-18T13:57:44Z","signature_id":"fef4719853b479dc28536b82a7d708acb33374f964d7d3bf781de8b061fdf911","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6586/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708355189739520","occurred_at":"2026-08-18T13:36:31Z","signature_id":"2b0907088bb4aac07ad378641d2888e70f80f231ad0de964bb3b76f7d4ec2fc2","pr_number":6586,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708056525934592","occurred_at":"2026-08-18T13:36:10Z","signature_id":"cdfd836f394f5ba29233bf3b5c663766097ad8fe21572b68c4346fcaaab2130e","pr_number":6585,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"aea70a0fde9c5036e02637cee20b6a7d41343b1f0c5870f8b5e76e81cd5e3e14","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089679732839813120","occurred_at":"2026-08-18T11:44:07Z","signature_id":"e7e1b7acc7dd0d1737cd278e8717b3ddb4590df9a25e3747507db1280822a053","pr_number":6553,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089665224855523328","occurred_at":"2026-08-18T10:46:28Z","signature_id":"e033375a35e19d9c580096c94daf90fc34c0dfbfaf3fe6a00ac93387acbe9f21","pr_number":6573,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089651610870353920","occurred_at":"2026-08-18T09:52:22Z","signature_id":"fd9b0b6481adc816048ae34a60453d0814a31d87ca09911c89a5cf1aa6c8b15b","pr_number":6562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089560445248606208","occurred_at":"2026-08-18T03:50:06Z","signature_id":"b6f5c2b9ba3c302beaf719d7d12b91d143aa8a193a07e20457dc65ab425f203a","pr_number":6566,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"258f96de493739fe955a1bf57e30a40daff50b064a34fc50c1905fdfb2068f03","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":58,"runs_affected":33,"failed_runs":33,"run_impact_percent":5.621805792163544,"anchor_occurrences":58,"after_last_push_count":16,"also_in":["INT","PROD","STG"],"trend_counts":[2,18,2,3,0,4,0,31],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":53,"prior_last_seen_at":"2026-08-22T18:55:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-h74snzg5tpbh/cluster-candidate-5-0-gp9ztx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014a05c0\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-gp9ztx: failed waiting for cluster=\"cluster-candidate-5-0-gp9ztx\" in resourcegroup=\"rg-candidate-5-0-h74snzg5tpbh\" to finish creating: GET https://rp.j1843200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de4739f4-eebd-4fe2-bf80-3bf6d57ff617\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de4739f4-eebd-4fe2-bf80-3bf6d57ff617\",\n      \"name\": \"de4739f4-eebd-4fe2-bf80-3bf6d57ff617\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T00:23:47.993778746Z\",\n      \"endTime\": \"2026-08-25T00:42:49.076566651Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-h74snzg5tpbh/cluster-candidate-5-0-gp9ztx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014a05c0\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-gp9ztx: failed waiting for cluster=\"cluster-candidate-5-0-gp9ztx\" in resourcegroup=\"rg-candidate-5-0-h74snzg5tpbh\" to finish creating: GET https://rp.j1843200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de4739f4-eebd-4fe2-bf80-3bf6d57ff617\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de4739f4-eebd-4fe2-bf80-3bf6d57ff617\",\n      \"name\": \"de4739f4-eebd-4fe2-bf80-3bf6d57ff617\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-25T00:23:47.993778746Z\",\n      \"endTime\": \"2026-08-25T00:42:49.076566651Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-x45vwlgpgvzk/cluster-candidate-4-21-rztpw4 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f3e6a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-rztpw4: failed waiting for cluster=\"cluster-candidate-4-21-rztpw4\" in resourcegroup=\"rg-candidate-4-21-x45vwlgpgvzk\" to finish creating: GET https://rp.j4709760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e986c420-0503-4186-9a01-bad1f92a005c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e986c420-0503-4186-9a01-bad1f92a005c\",\n      \"name\": \"e986c420-0503-4186-9a01-bad1f92a005c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:37:07.202301694Z\",\n      \"endTime\": \"2026-08-24T23:56:10.677598487Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-x45vwlgpgvzk/cluster-candidate-4-21-rztpw4 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f3e6a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-rztpw4: failed waiting for cluster=\"cluster-candidate-4-21-rztpw4\" in resourcegroup=\"rg-candidate-4-21-x45vwlgpgvzk\" to finish creating: GET https://rp.j4709760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e986c420-0503-4186-9a01-bad1f92a005c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e986c420-0503-4186-9a01-bad1f92a005c\",\n      \"name\": \"e986c420-0503-4186-9a01-bad1f92a005c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:37:07.202301694Z\",\n      \"endTime\": \"2026-08-24T23:56:10.677598487Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d48840\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-mglhphvdtjhd\" to finish creating: GET https://rp.j6339200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1dbfdf90-d2f5-4efc-b68b-42b49d8220eb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1dbfdf90-d2f5-4efc-b68b-42b49d8220eb\",\n      \"name\": \"1dbfdf90-d2f5-4efc-b68b-42b49d8220eb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:29:18.405302537Z\",\n      \"endTime\": \"2026-08-24T23:48:26.458954406Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d48840\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-mglhphvdtjhd\" to finish creating: GET https://rp.j6339200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1dbfdf90-d2f5-4efc-b68b-42b49d8220eb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1dbfdf90-d2f5-4efc-b68b-42b49d8220eb\",\n      \"name\": \"1dbfdf90-d2f5-4efc-b68b-42b49d8220eb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:29:18.405302537Z\",\n      \"endTime\": \"2026-08-24T23:48:26.458954406Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092034671041843200","occurred_at":"2026-08-24T23:41:48Z","signature_id":"54a530c9876fc008fec6ac6899f443f3e3f5628a64422efb1e6064e8186af6e1","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092020242174709760","occurred_at":"2026-08-24T22:44:28Z","signature_id":"e6a9db1c5a3cafd8f545d8476a1e6b49dd9166a98aa20d59709936d40b1c52c0","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6663/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092019318316339200","occurred_at":"2026-08-24T22:40:47Z","signature_id":"a6a8523a32b9fba91a90fbd04abe90465e11b86cdf06949b929a37a85e4f40ad","pr_number":6663,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6663/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092019318316339200","occurred_at":"2026-08-24T22:40:47Z","signature_id":"e6285c4c12e9e0c85b5ecc25058de40b9f05b37d1a08c8a8c4a66108766b8ee0","pr_number":6663,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092016236664721408","occurred_at":"2026-08-24T22:28:33Z","signature_id":"4549c28d9f346a4101b2d199c56363a0a36787b6952f32440182d5210d9903d4","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092016236664721408","occurred_at":"2026-08-24T22:28:33Z","signature_id":"916b71cbe1c0282bebb7706c18cf185456f94291c6c7059f8d673bee6920e761","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"49911eb59aa73a3d7ebf02bdb4e56de7a616897c72e3ac7abe420c0c7a47e40e","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"6487190a1499ee6eb043aa40a245add60c110f31922a104a2b7ccae5c75948f8","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"7cc1252e9b8fe966bbe77836b76cbb84f92a735b546bfadc6681cd7399872b22","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"a9397a1f0aaba5d8ad982fe383aca1267736bcbd0d40f55741f427e85ee8b6d8","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"c5d8b0e5c56e2c31565dc9e53627bd3252420239c844111c2d9db849f9e9c51c","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"deae4888a32973190e55fe6f1f2ac8a8941dd6efae21aa17eaa50cc4448f5b70","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","signature_id":"7451289a87442818c6daa03144d0fb9efebafa88d3687d31f99e8439ca741e97","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091965378614792192","occurred_at":"2026-08-24T19:06:19Z","signature_id":"7830c998ab91a0251740ef30429a36f78ee908a1781d172a1f172ebef622c56d","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091963250454302720","occurred_at":"2026-08-24T18:58:00Z","signature_id":"75d7cd315bc777c1079b9374c7a2b4aabd61d42a04817f2523f7ea7dab1326a9","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","signature_id":"2021d27012220f14a5eaeb1e574cf0e0cf00596ef715d56d7ab033d1204d0ed5","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","signature_id":"dea6785389a59c2a975d6ddfe69a85996e881ce704015e7a9c41548aeaca0017","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091939168878858240","occurred_at":"2026-08-24T17:22:18Z","signature_id":"19ba2327e177a466c74ef48f9d128288b644237f0c8199be7309ae0bf40f5a82","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091932257060130816","occurred_at":"2026-08-24T16:54:39Z","signature_id":"70c12268048fb1a990e1fcfe48d8d53530e6791148ebcbe91068d54355e50247","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"688dcfad091ac3bb942f27fcd5e2e3d6471f49c9042338c1c965ab6c339ae7e6","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"d1005ca6dc326e3c744c86ae1e9fb1086de2e12e4a6351183145527b31e5eccf","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"db07fb00cb7c1dc1d49041e7a151eab39b4fee2bc9901dd00a1fc182625826d2","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"40b147b3a09185f7fbbd5416adf8d0e158c67796d5387c2aa465c0a43fd1dc30","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"8d53ed6b1258502dfa6cecd4e6049f72772418b7686ae5c9e29c6adc65e73c79","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091899442289446912","occurred_at":"2026-08-24T14:44:27Z","signature_id":"fd285a3626d258dcb1ba2d99b9eb4762dc5e5af32183bf400bd799c872dd2fda","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","signature_id":"94f9ba782dfa48d054a8eb8de04c3bde301c62297127cae4d03aeb5f65b1b3b4","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","signature_id":"e6714abbbcfe1d54c45ec38583b6367bc1667d055485de6ce77544f68acc9679","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895460682797056","occurred_at":"2026-08-24T14:28:36Z","signature_id":"ccd1c15e8b2737a8cbb30d07ada8f1b6f0748afd9d43b748ec4128f7a61ce044","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879954055696384","occurred_at":"2026-08-24T13:27:00Z","signature_id":"b66537b24ffd588d399e5e7cc810bf3a188e5e0dce669b0b16956a016a18a10c","pr_number":6429,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879927736438784","occurred_at":"2026-08-24T13:26:52Z","signature_id":"6a060f99c27004cf01810556d4b6e1126f6aeee13d0cc7b3c046327b6e9bcc0d","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091855258673221632","occurred_at":"2026-08-24T11:48:53Z","signature_id":"25baffda5fc6f88c385cae06e401a6f4815895c798a9d1e087a18364db777535","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6555/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091237855425728512","occurred_at":"2026-08-22T18:55:31Z","signature_id":"df6c1e4a9641386a7eb5a26baa92084f62c849def92ac231cd5d430c67b19146","pr_number":6555,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091230843602210816","occurred_at":"2026-08-22T18:27:40Z","signature_id":"0420a23f54b484bd5c42b458707ed3ab40693ba805498945d41e0e64e6ee0507","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091230843602210816","occurred_at":"2026-08-22T18:27:40Z","signature_id":"1f603ba8c82583eec0f2843da23cf36993226ade45076c14bee8cc9c3b4c788e","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091230843602210816","occurred_at":"2026-08-22T18:27:40Z","signature_id":"9534a025ea1fda5ac6df21102fd48d9c182b38eaf8806344cd04c3af0afa585d","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090577382321688576","occurred_at":"2026-08-20T23:10:58Z","signature_id":"5715aefa195ea2179c48b8cdb3c0646c0b4dd4ee7db7b4e95085b994ec083207","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090554196767019008","occurred_at":"2026-08-20T21:38:55Z","signature_id":"3841200b7547f214afafd0c0c55361fa1f72cacddfaad20a9fcd5f4fb1711cd9","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6555/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090233576732758016","occurred_at":"2026-08-20T00:24:53Z","signature_id":"b4753d3fccec6f82fdb286c98c9fe8175786c7811cebd07b1f3eec79c958bbe0","pr_number":6555,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090166116277030912","occurred_at":"2026-08-19T19:56:50Z","signature_id":"d60a5810e046e63b38869d3666f37ae08c3d445efbce501ba3a5a908dfb76790","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090093258611888128","occurred_at":"2026-08-19T15:07:19Z","signature_id":"dbcf091eba0b35144538bee3334ec2a7d62e240fcf6f751b27ae3686de0d02e8","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6581/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089746333521940480","occurred_at":"2026-08-18T16:08:46Z","signature_id":"2f3b36c7b179297e4c66e80840b6e53d54d2c59665cc143f7698e0a6abba6535","pr_number":6581,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6564/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725877364461568","occurred_at":"2026-08-18T14:47:25Z","signature_id":"3ee1bf6c2207ae7870767eb50e42ffe4c968d019f54e37a385dc742b1f01e516","pr_number":6564,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089723779814002688","occurred_at":"2026-08-18T14:39:04Z","signature_id":"5afdffc644d3a5d7f8e99585ab0d51910f6c1a955047a5104ec5b5d0dc01ab5c","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089690861355929600","occurred_at":"2026-08-18T12:28:20Z","signature_id":"0771cc3fd72fc1a9caf13a9f23270a8d1d91cbdc1f049a437a628ba4793681d4","pr_number":6564,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089690861355929600","occurred_at":"2026-08-18T12:28:20Z","signature_id":"1bf6b56d2abcf84bae5e2fdaa0ff96583ab3513311cf6922b01e1da3c9118db4","pr_number":6564,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089690861355929600","occurred_at":"2026-08-18T12:28:20Z","signature_id":"4f0d902a71df7f599f939a3799bab0f4bc88d79690fb832cbcf06a9ab5518ea1","pr_number":6564,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089690861355929600","occurred_at":"2026-08-18T12:28:20Z","signature_id":"553a2980fc369edc50b9b0874c636c698652138b290a710d3a10260b72668db7","pr_number":6564,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089690861355929600","occurred_at":"2026-08-18T12:28:20Z","signature_id":"5b937940fd96caf141466060200e4d8b907df3916d919b98a2710754fc4bd61d","pr_number":6564,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089690861355929600","occurred_at":"2026-08-18T12:28:20Z","signature_id":"93bc97b8285b2048f5cc71559c82f7483590fa8bf8397d8a260cb922f3a3f7f0","pr_number":6564,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089690861355929600","occurred_at":"2026-08-18T12:28:20Z","signature_id":"f3a005f818eaa456cdf62739926951450f1cfa8314a8cb4ace281eff44b632dc","pr_number":6564,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"129b444185ef10b7f9868581dc8f1b1d91b4c4cfc889c0906c282b4686140c74","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"45c750ae7857adeb81b44d2dbf24c26fab84cd48c478ba87ef95b57e4e7d2f46","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"b73877bb1beeb51920250cb3a871d892f380c55adbd2e87d9cb15cb870d4970a","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"beb7703b2ebd482c49c85dd544476422876c8635f9868443f3653374e894153e","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"e73b54f0ff5003d50dc0d106ada3d6cd8cbfc4c870a609649c3c63cb2dacfce2","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"f26abf00c2d31a76583a85f15193ca876f3c58818aac99e929877721252c448b","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089679732839813120","occurred_at":"2026-08-18T11:44:07Z","signature_id":"315349b476595d08890f02fa76d28d83683b4a316f2bc010f71e62ed064ae352","pr_number":6553,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089679732839813120","occurred_at":"2026-08-18T11:44:07Z","signature_id":"d62007d82759d307045ebe65d9c2cbcd1b766aba9fb8b3d3b0cea02236c50cdf","pr_number":6553,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":53,"runs_affected":31,"failed_runs":31,"run_impact_percent":5.281090289608177,"anchor_occurrences":53,"after_last_push_count":117,"also_in":["INT","PROD","STG"],"trend_counts":[23,17,12,13,10,1,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":124,"prior_last_seen_at":"2026-08-22T00:13:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":7},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001bdc3d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-k276rwlx6nvv\" to finish deleting: GET https://rp.j4608896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95be590e-be2d-4405-b000-e47e729c722a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95be590e-be2d-4405-b000-e47e729c722a\",\n      \"name\": \"95be590e-be2d-4405-b000-e47e729c722a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:29:12.661843478Z\",\n      \"endTime\": \"2026-08-22T01:53:12.268348605Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sbairsosicqjiq127hgv6ukup2i9asv still exists (deletion dispatched at 2026-08-22T01:29:12Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001bdc3d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-k276rwlx6nvv\" to finish deleting: GET https://rp.j4608896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95be590e-be2d-4405-b000-e47e729c722a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/95be590e-be2d-4405-b000-e47e729c722a\",\n      \"name\": \"95be590e-be2d-4405-b000-e47e729c722a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:29:12.661843478Z\",\n      \"endTime\": \"2026-08-22T01:53:12.268348605Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sbairsosicqjiq127hgv6ukup2i9asv still exists (deletion dispatched at 2026-08-22T01:29:12Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc002b983a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-8fhdctfdz8rw\" to finish deleting: GET https://rp.j0088320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1d2b6d51-1179-46e6-9785-09bffda18aba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1d2b6d51-1179-46e6-9785-09bffda18aba\",\n      \"name\": \"1d2b6d51-1179-46e6-9785-09bffda18aba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T22:37:19.756260961Z\",\n      \"endTime\": \"2026-08-21T23:01:22.169782395Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sb7u2scpkh7308u4g26d4adn4v6f2fq still exists (deletion dispatched at 2026-08-21T22:37:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc002b983a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-8fhdctfdz8rw\" to finish deleting: GET https://rp.j0088320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1d2b6d51-1179-46e6-9785-09bffda18aba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1d2b6d51-1179-46e6-9785-09bffda18aba\",\n      \"name\": \"1d2b6d51-1179-46e6-9785-09bffda18aba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T22:37:19.756260961Z\",\n      \"endTime\": \"2026-08-21T23:01:22.169782395Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sb7u2scpkh7308u4g26d4adn4v6f2fq still exists (deletion dispatched at 2026-08-21T22:37:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00160c810\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-7ckkp246v7kn\" to finish deleting: GET https://rp.j0088320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/daa2ac24-e65d-4ec6-b703-b65cb92de73e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/daa2ac24-e65d-4ec6-b703-b65cb92de73e\",\n      \"name\": \"daa2ac24-e65d-4ec6-b703-b65cb92de73e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T22:26:31.187721464Z\",\n      \"endTime\": \"2026-08-21T22:50:32.176977266Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sb7tr4tpdqmibta8tjhvqb9049in5ek still exists (deletion dispatched at 2026-08-21T22:26:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00160c810\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-7ckkp246v7kn\" to finish deleting: GET https://rp.j0088320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/daa2ac24-e65d-4ec6-b703-b65cb92de73e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/daa2ac24-e65d-4ec6-b703-b65cb92de73e\",\n      \"name\": \"daa2ac24-e65d-4ec6-b703-b65cb92de73e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T22:26:31.187721464Z\",\n      \"endTime\": \"2026-08-21T22:50:32.176977266Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sb7tr4tpdqmibta8tjhvqb9049in5ek still exists (deletion dispatched at 2026-08-21T22:26:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6593/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090955578854608896","occurred_at":"2026-08-22T00:13:52Z","signature_id":"ea71a88634a16d0afbc9df293cf4726a2f2149536108ac3773b3c1495ab52cda","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090913249020088320","occurred_at":"2026-08-21T21:25:40Z","signature_id":"9cb8bbbd46d4ff7b8d8a9145481c0d6214ebf1fee2dd9871b79584eecc0ace23","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090913249020088320","occurred_at":"2026-08-21T21:25:40Z","signature_id":"ed56ae2522d4aa5d67496604431cbfa03d468518851d39e15f802a1713c02c70","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090912033770835968","occurred_at":"2026-08-21T21:20:50Z","signature_id":"c9b7084d1a6d0d9bf7767d41e2dffc0fe9a0c368f17fbd2ba52ca28c7fef9528","pr_number":6651,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090849031038177280","occurred_at":"2026-08-21T17:10:29Z","signature_id":"28c8a564f7881af385b8b7f9b5484f7b699520a4ab948ec6980718c6018c0c72","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"78a303c77738b363a2fb051a390459c944a6c6506892466069ef310d887a73bf","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"e8da6919ba5cfbd1fb3e43dfe2e38e3fb9f1811073e2681ed20f13aaa672f01d","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"f7778fa6694474ab43ef140bac6b29ba7b40c11ac312df889617c83f251ff3e8","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657643948412928","occurred_at":"2026-08-21T04:29:59Z","signature_id":"25f5d6f9a04ec10b0a212be6cba0955c5609c0d0248a7038cbecb0bdeb77becf","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"b6e0d020ce7138aead9b3214d404255ccec60fbcbcf4cf08b2dceeef66ee2cb5","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090612430223708160","occurred_at":"2026-08-21T01:30:11Z","signature_id":"6f65a26c62cfa1ddfc52a9edb6e78c7e84f29cf938bdb44ba828a131a29ffb1b","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"526dfc672e25122e7f8723ddcc2e06db50b45870112cb9c196a9ec2b74c0177e","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090497381098328064","occurred_at":"2026-08-20T17:53:09Z","signature_id":"669b518bb8bb2772c1633069821e688c83a58a278e8614898aa7d5d3a3893821","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"89c0ce8eaad0a52018f4811492cf780d403fc9c453f7b4120a99286a662876fa","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"8ec48adca2936da1cda6f46dd400d943903ebe2820f6d5d048144f838fb306bf","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"c0b52de30405f1ce85289aff26156b7972cd83d819d41fc68c0f6d416896168c","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"eaa451d983ea15579a77a41e048b7449200983b60173dcef401970a8df5f49b6","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"f186ce9f560dda14bcb06e1c713c1ef4f35902f5b2c4080bc81055330cea6fa1","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090473213871001600","occurred_at":"2026-08-20T16:17:01Z","signature_id":"7303da35a96fbcfc9b1e7b5f88ae3374d9ec8cfc0a32609b0d1207bd56d810ad","pr_number":6621,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090473213871001600","occurred_at":"2026-08-20T16:17:01Z","signature_id":"db4ffd88711ea8acaa394bcc753b2c312b02ba7b4aa456f3a34b6a04f2f7399e","pr_number":6621,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"6ffc943b94f0cf1e4d54823cd500d26e42b1dd3574092ad832c9467d03061c46","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"280c80547a16d77efdca2f79dc8afee598695f38c3aeaa0d818b0900296fa51a","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"3da52cfa371882cf7ceec3946de980d3513c1eb93b2b9ec3854cf2a632d3cd96","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"a96fcc5ef3389bbac06e463dd369b5888c6f3ff0d594f053e80261667944cc1c","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"4a72bbb712e2337ea1052f6b5b96c57b641dda1c52686aa38e3b945cc76b0c0a","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"7065b6a303a56f0747f08761d1761fb4fe32033ebd0de9f2a8c85471aeb8ea96","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"e882de52842635c93d4cef7d33a3a311986afa2764942b5596deabc845a3a945","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090111171486027776","occurred_at":"2026-08-19T16:18:30Z","signature_id":"f0ad9c22bf6fa48d6421f1b3522dd36f34546531d81a66e4fccb8efb04ee61cd","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090087137016811520","occurred_at":"2026-08-19T14:42:58Z","signature_id":"ddc579d2118f83fc570d9a2f19f7609d1aec309883d03b2d39212f6f1f3543ce","pr_number":6553,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090079394612645888","occurred_at":"2026-08-19T14:12:08Z","signature_id":"337286aa4213d0c7cc64473c7e3e34c20f6acd5171260d91c52af41491d1f43b","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090079394612645888","occurred_at":"2026-08-19T14:12:08Z","signature_id":"a68839bcf6dfec9c97e979fa1937e27adc5a54ca6b682fd582d1dcc41b100773","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090035242617278464","occurred_at":"2026-08-19T11:16:37Z","signature_id":"4de85c8d82eaaea68218d8d636d979c1c2ec23176f984d427e09c48365fa8ec8","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090035242617278464","occurred_at":"2026-08-19T11:16:37Z","signature_id":"6bd6698f03122a7286ece459cd702827041a0d6998cdc483c51d2cc13c809912","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"07928273367f3cea76524c08cc39626ec0bb68cbd2c65cc52d78457055bc9d40","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"495783b014e34adc3be2999cc227819cc07e808e7fc136b141dce817a9754c46","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"c62c14d54dade7c8d4c702367d469896ff1ea7b078ac4a9da2f5f1b0df6c564b","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089850706507862016","occurred_at":"2026-08-18T23:03:30Z","signature_id":"3357dc01711efce6afb8b8e256eda8bda3c7b14646a4ca59df769f044005ae89","pr_number":6132,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089850706507862016","occurred_at":"2026-08-18T23:03:30Z","signature_id":"96616559ea273ac32162b57583e6409fc8868359668578c970d9dc32e811873c","pr_number":6132,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6594/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089843707992870912","occurred_at":"2026-08-18T22:35:42Z","signature_id":"5e509d4f904fb4c81d9b8ed9730cea9041e2f0c772fc297580313e2eec2b2a4b","pr_number":6594,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089829163719462912","occurred_at":"2026-08-18T21:37:54Z","signature_id":"03094c5eff6a55bb039b9d2b8800a8bd980196c0a769d6e77a53e340650ab4ab","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089809279165927424","occurred_at":"2026-08-18T20:18:53Z","signature_id":"2ad5c26dee74eff0427c294fbf1271eee713558b82a146e6216fe9b9b1d9e2f3","pr_number":6562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089809279165927424","occurred_at":"2026-08-18T20:18:53Z","signature_id":"43ebacff76c499010f64232baea9066fb9a4c5a3bd56909027e4b996e92f3608","pr_number":6562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089809279165927424","occurred_at":"2026-08-18T20:18:53Z","signature_id":"9edabaa1872ee021a00fad3a753e4c5e95185838dffbd0ab28258bb0cf4479a3","pr_number":6562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089757851600818176","occurred_at":"2026-08-18T16:54:32Z","signature_id":"5edfb45d29686d9fa30473b2ab49fcc2939f158644857a45002401ed4d1adfde","pr_number":5880,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089753182908452864","occurred_at":"2026-08-18T16:35:55Z","signature_id":"ea14c204a1ba23d92da42e884f6d3fc61857a229dd405073e7d808500f70a15a","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089723779814002688","occurred_at":"2026-08-18T14:39:04Z","signature_id":"3a4c056ea0c7b567b9b835373634a84e199c080de7418a24dc145a53a23427b3","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089723779814002688","occurred_at":"2026-08-18T14:39:04Z","signature_id":"a8aa174a097e7e748b00dd4dace4b6e8b5e236cf10bc9c351f54b7f8606e7030","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708056525934592","occurred_at":"2026-08-18T13:36:10Z","signature_id":"b005866e125da627c98eefd71c181b97827d93dd930ded3c58cc67fb619b9316","pr_number":6585,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6426/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089706907987087360","occurred_at":"2026-08-18T13:32:06Z","signature_id":"02b6f30f827bbdff486c25853675ca8c1274c8d525e4d4e83ef1f584fddc74da","pr_number":6426,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089686738145382400","occurred_at":"2026-08-18T12:11:57Z","signature_id":"7508c6c2f50ff79d5a1064193a2f7bc08f13f87e0e0de61fc584f63b79fd1e10","pr_number":5880,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089665224855523328","occurred_at":"2026-08-18T10:46:28Z","signature_id":"63f0e1595f6879da3a45b881a9dd72ff5ac8a30f793e9c071f2a144e45e018d3","pr_number":6573,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089665224855523328","occurred_at":"2026-08-18T10:46:28Z","signature_id":"7681d0c1e854e60c49c9ad4ecff68ac045f97c81b1b64c579ba66a425117e7a1","pr_number":6573,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089665224855523328","occurred_at":"2026-08-18T10:46:28Z","signature_id":"8deb034a5d2b9bf086be517e01aa787f2160c7ba328795567ba98a16efba08d9","pr_number":6573,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":49,"runs_affected":49,"failed_runs":49,"run_impact_percent":8.347529812606474,"anchor_occurrences":49,"after_last_push_count":69,"trend_counts":[16,15,12,10,11,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":228,"prior_last_seen_at":"2026-08-21T21:25:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":49}],"full_error_samples":["alert fired 24 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T16:11:08Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"kube-proxy\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/kube-proxy has not finished or progressed for at least 15 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"cloud-node-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/cloud-node-manager has not finished or progressed for at least 15 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azuredisk-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azuredisk-node has not finished or progressed for at least 15 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:25:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azurefile-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azurefile-node has not finished or progressed for at least 15 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-provider-azure\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-provider-azure has not finished or progressed for at least 15 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-npm\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-npm has not finished or progressed for at least 15 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-08-24T16:13:07Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-provider-azure\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-provider-azure has not finished or progressed for at least 15 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-08-24T16:13:07Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azurefile-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azurefile-node has not finished or progressed for at least 15 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azuredisk-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azuredisk-node has not finished or progressed for at least 15 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"kube-proxy\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/kube-proxy has not finished or progressed for at least 15 minutes.\nFiring 17:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:26:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 18:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-npm\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-npm has not finished or progressed for at least 15 minutes.\nFiring 19:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:24:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 20:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"cloud-node-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/cloud-node-manager has not finished or progressed for at least 15 minutes.\nFiring 21:\n  State:       Resolved\n  Started:     2026-08-24T16:17:09Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 22:\n  State:       Resolved\n  Started:     2026-08-24T16:17:09Z\n  Ended:       2026-08-24T16:23:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet velero/node-agent has not finished or progressed for at least 15 minutes.\nFiring 23:\n  State:       Resolved\n  Started:     2026-08-24T16:17:09Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet velero/node-agent has not finished or progressed for at least 15 minutes.\nFiring 24:\n  State:       Resolved\n  Started:     2026-08-24T16:17:09Z\n  Ended:       2026-08-24T16:29:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","alert fired 24 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-provider-azure\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-provider-azure has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azurefile-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azurefile-node has not finished or progressed for at least 15 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-npm\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-npm has not finished or progressed for at least 15 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"cloud-node-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/cloud-node-manager has not finished or progressed for at least 15 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azuredisk-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azuredisk-node has not finished or progressed for at least 15 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"kube-proxy\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/kube-proxy has not finished or progressed for at least 15 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"cloud-node-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/cloud-node-manager has not finished or progressed for at least 15 minutes.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azurefile-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azurefile-node has not finished or progressed for at least 15 minutes.\nFiring 13:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 14:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-provider-azure\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-provider-azure has not finished or progressed for at least 15 minutes.\nFiring 15:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-npm\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-npm has not finished or progressed for at least 15 minutes.\nFiring 16:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azuredisk-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azuredisk-node has not finished or progressed for at least 15 minutes.\nFiring 17:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 18:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 19:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 20:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"kube-proxy\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/kube-proxy has not finished or progressed for at least 15 minutes.\nFiring 21:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:53:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 22:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:53:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet velero/node-agent has not finished or progressed for at least 15 minutes.\nFiring 23:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:53:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet velero/node-agent has not finished or progressed for at least 15 minutes.\nFiring 24:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:53:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.","alert fired 24 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-provider-azure\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-provider-azure has not finished or progressed for at least 15 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"kube-proxy\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/kube-proxy has not finished or progressed for at least 15 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azuredisk-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azuredisk-node has not finished or progressed for at least 15 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-provider-azure\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-provider-azure has not finished or progressed for at least 15 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"ama-metrics-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/ama-metrics-node has not finished or progressed for at least 15 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-npm\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-npm has not finished or progressed for at least 15 minutes.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azurefile-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azurefile-node has not finished or progressed for at least 15 minutes.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"aks-secrets-store-csi-driver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/aks-secrets-store-csi-driver has not finished or progressed for at least 15 minutes.\nFiring 13:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"retina-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/retina-agent has not finished or progressed for at least 15 minutes.\nFiring 14:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"cloud-node-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/cloud-node-manager has not finished or progressed for at least 15 minutes.\nFiring 15:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-npm\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-npm has not finished or progressed for at least 15 minutes.\nFiring 16:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azurefile-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azurefile-node has not finished or progressed for at least 15 minutes.\nFiring 17:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"kube-proxy\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/kube-proxy has not finished or progressed for at least 15 minutes.\nFiring 18:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 19:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"csi-azuredisk-node\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/csi-azuredisk-node has not finished or progressed for at least 15 minutes.\nFiring 20:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"cloud-node-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet kube-system/cloud-node-manager has not finished or progressed for at least 15 minutes.\nFiring 21:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet velero/node-agent has not finished or progressed for at least 15 minutes.\nFiring 22:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 23:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:52:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 24:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:52:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet velero/node-agent has not finished or progressed for at least 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"367941eb104ee306ff1fe196c961bc8c4a584d69e0f31a0ee0696e981fbdf639","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090913249020088320","occurred_at":"2026-08-21T21:25:40Z","signature_id":"83f1bad8d518c7beb63bce55e051860d194f93a05a39bd45a36f26ea11c37de9","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090912033770835968","occurred_at":"2026-08-21T21:20:50Z","signature_id":"493cc43ac674b927d12a09d7344bb8455bf9bb59b8b34c2ade48063df28c770c","pr_number":6651,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894675127832576","occurred_at":"2026-08-21T20:11:52Z","signature_id":"4b5ff8ddc74fbca1ad06f8acb960b40e5ac5288bdc02906474049eec047acbac","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090849031038177280","occurred_at":"2026-08-21T17:10:29Z","signature_id":"54718b687d279cfeef25d73a1bd15f4acbe2c392068026450ee15a69c107561e","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"2c84bb7c1b1025bf5aca49043832b9795d9a698197c24f25788bf11286858928","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090676319984553984","occurred_at":"2026-08-21T05:44:12Z","signature_id":"c9a68fbb944333ee25a753b5b0f81d4460bb338db83839755dd4687597170d5d","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090660118717796352","occurred_at":"2026-08-21T04:39:49Z","signature_id":"2618f72610682f43dd2cd2dba26bdc0127c8d0b0defed2ee2da18c76b6b9077d","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657643948412928","occurred_at":"2026-08-21T04:29:59Z","signature_id":"edd75028aec82bdb3a00ffa4b080650ca91380325a410deca047f6956533bdff","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"85e7b46b9b7ff85d3a77c82be204f3e8ee605ac388a88750102a74b6fcfc8e11","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090643626764800000","occurred_at":"2026-08-21T03:34:17Z","signature_id":"c94a19cdb1a31ba9664c24415b96a5205d3ca0629c25480f0e83765cac75c579","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090612430223708160","occurred_at":"2026-08-21T01:30:11Z","signature_id":"bc8a93b0db0fe1ef3d350779090613683a43a036387d73cd70b9d66fe288f8ec","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090587456947621888","occurred_at":"2026-08-20T23:51:05Z","signature_id":"2991094f0824a50ebc0d788dd77119799c574f1dac8d1b4357e9ec4e67b7f828","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"d15020d9116463034a4be5a9d6d97582c79c8f179dc8049587e89517faf78588","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090497381098328064","occurred_at":"2026-08-20T17:53:09Z","signature_id":"9f4f056083aa68743b18cd9abb3eff1c8907f5504b1ce08ae3c69018666a2320","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"8b71cdf7aa2dd4cf64a1f93522e60dad9e67f08b57d138afe592dcb0edfaf800","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090473213871001600","occurred_at":"2026-08-20T16:17:01Z","signature_id":"1f2d648dc953e3539e09c6b34321427152afbeb6e7be3ea9fe5c07b1939bdd44","pr_number":6621,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090443849187987456","occurred_at":"2026-08-20T14:20:26Z","signature_id":"36b8f686529afbb4e5980b5c913953298961d9ffb6b9e06ed78dea224b732056","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090419688172425216","occurred_at":"2026-08-20T12:44:10Z","signature_id":"52ba71bccdd9820d2abd1b0c61181d75f220e8e4f06edd79e92c083588e7f3f2","pr_number":6617,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6589/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090373215313465344","occurred_at":"2026-08-20T09:39:46Z","signature_id":"5fe4bd31ea1c475a5ead0034adafd043b80ceb7a26a5465a58e37af7a35a146d","pr_number":6589,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"36f2100c2a1fe2d84fdef00ccf37bdcf7e79999878e0a2391eb45841bd6c6a3b","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"02e6579931a5425586fd1b99e79cb9403b00e6ad53c42c0f54c14a97d68e75c2","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090180258316685312","occurred_at":"2026-08-19T20:53:01Z","signature_id":"8e9b9cfa88833157f1361b4031ce44e1106e86384c5dbf97963b200765db60f3","pr_number":6614,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"b3b7a82dd8fc7c9453c13d5bac57f70bad1257901992ef7e3b00a768cf93ce29","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090126361980047360","occurred_at":"2026-08-19T17:18:51Z","signature_id":"5b7f46f1c74c9f7df302416b3ff8d6f9c3b15b2f31197a771207b8031862e1a6","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090119905717260288","occurred_at":"2026-08-19T16:53:12Z","signature_id":"4a381fdecdd0a4be7797b7b05cb5a2e67ba5334f09d2a876403d68397bddc7b4","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090111171486027776","occurred_at":"2026-08-19T16:18:30Z","signature_id":"536e9b5133376da7186a6910260a4ff7eb9410c6c312609565848cec9c00b9f9","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090087196852752384","occurred_at":"2026-08-19T14:43:14Z","signature_id":"c8313ef1a30fd3399e30c4c34f4f78d6f3cd827c632099807a4c65aeed41251e","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090087137016811520","occurred_at":"2026-08-19T14:42:58Z","signature_id":"b09c2b57753187375e9bb4bfbf655cf935649a379ae9145ab6a1173455b81add","pr_number":6553,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"6d5c6eda78682131a737b5a1673e8cff260b18e1476bf9fc6e7786614a2c8ea8","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090072497390096384","occurred_at":"2026-08-19T13:44:49Z","signature_id":"b3476258f174db773fca00d2a8a1840bd4d80a888f87e9c93bf4175b85362c15","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090035242617278464","occurred_at":"2026-08-19T11:16:37Z","signature_id":"f6b16d89151fe9aa188d216d1e54e2be6e1547185a4d587433d32485b63925b8","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090031884472946688","occurred_at":"2026-08-19T11:03:26Z","signature_id":"20f3b15ee34600ced7a0a232da78a03021e68bcf501decf6ca03614b467445e0","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"ee274618c50bb1e5934e1429251a60e64d1ea78f9e670a0c1bef6baea15b5b1b","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089829163719462912","occurred_at":"2026-08-18T21:37:54Z","signature_id":"49f018227e722ec5078f1ddda97432728aaaaac214c7fd1cc6a7eb84ff48bc98","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089809279165927424","occurred_at":"2026-08-18T20:18:53Z","signature_id":"ba83cbbb37c7ba19baa21bc52f8f076e4c0ac11a6b36a47b0891581405bcc56f","pr_number":6562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089796885723222016","occurred_at":"2026-08-18T19:29:38Z","signature_id":"8f2c00aabe605ded4e8a92d94cc9db3a37da95aaef564345b2467361fa4d4470","pr_number":6592,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089768451781431296","occurred_at":"2026-08-18T17:36:39Z","signature_id":"af1f018d57a18cc3b283836e9a76596a177d7112c5c20bc0d248966d848c921d","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089759198756737024","occurred_at":"2026-08-18T16:59:53Z","signature_id":"d2b7bcf050acba6c9e2f2e31fc57c28dc694a69611323fe57e10893952595b15","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089757851600818176","occurred_at":"2026-08-18T16:54:32Z","signature_id":"da35e433f9379f807d3b1a23ef805d6c788b99b5f297c361c95d0070158b5bb2","pr_number":5880,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089753182908452864","occurred_at":"2026-08-18T16:35:55Z","signature_id":"5024c75807c14abb01222321752bc6907019c4c3330978d10befe11b78ccbce5","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"1a8f9e735780c563cc93041baef11a7b5889185144507f3e89674976fac7510e","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089723779814002688","occurred_at":"2026-08-18T14:39:04Z","signature_id":"e2de60eaa8eb87f2fe44c6cb700b00d9df2941c2603537579f4c9f2953c7dc06","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089713360550498304","occurred_at":"2026-08-18T13:57:44Z","signature_id":"b0b6b93600829297f67e8a0ed3503a32b3b3bb73b36ef807ded77184f6a2b9a4","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708056525934592","occurred_at":"2026-08-18T13:36:10Z","signature_id":"cc2da170d20eca1546e4a94bdb5453073389e71f2832c3aa9bd474cb965799e5","pr_number":6585,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6426/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089706907987087360","occurred_at":"2026-08-18T13:32:06Z","signature_id":"4716dd5431c5e693de773ac049d4f236917ced4937c86df2f2c314b51e020e83","pr_number":6426,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089686738145382400","occurred_at":"2026-08-18T12:11:57Z","signature_id":"d464d0e4532b985588e67970b1f86cbeecafdebfd94d885c9747315b5522eab0","pr_number":5880,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"16afc662dda6b16949fc1ff70c165c9876cd24911b8aa6efc8354e034ddd9d56","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089665224855523328","occurred_at":"2026-08-18T10:46:28Z","signature_id":"1eb3a5c963f58bfc015dd565bc8b8f9bd7270d35492773ac23bbd940f89496e2","pr_number":6573,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":44,"runs_affected":44,"failed_runs":44,"run_impact_percent":7.4957410562180575,"anchor_occurrences":44,"after_last_push_count":10,"trend_counts":[1,0,0,0,41,0,0,3],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":82,"prior_last_seen_at":"2026-08-21T20:12:08Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":44}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-25T00:10:04Z\n  Ended:       2026-08-25T00:34:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j8026752-svc\", component=\"backend\", name=\"observemanagedresourcegroup\", severity=\"warning\"\n  Description: Backend controller workqueue observemanagedresourcegroup has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T22:22:12Z\n  Ended:       2026-08-24T22:46:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j2642688-svc\", component=\"backend\", name=\"observemanagedresourcegroup\", severity=\"warning\"\n  Description: Backend controller workqueue observemanagedresourcegroup has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T19:54:46Z\n  Ended:       2026-08-24T20:17:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j4068096-svc\", component=\"backend\", name=\"observemanagedresourcegroup\", severity=\"warning\"\n  Description: Backend controller workqueue observemanagedresourcegroup has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092031971118026752","occurred_at":"2026-08-24T23:30:58Z","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091995273302642688","occurred_at":"2026-08-24T21:05:15Z","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","signature_id":"730b574e082ddd2b02cb621be4f7f301ab3ece9249671e4ee87a4e9243253f64","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894768476262400","occurred_at":"2026-08-21T20:12:08Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894756736405504","occurred_at":"2026-08-21T20:12:05Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894675127832576","occurred_at":"2026-08-21T20:11:52Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090878713473273856","occurred_at":"2026-08-21T19:08:16Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090877968120287232","occurred_at":"2026-08-21T19:04:55Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090871776832655360","occurred_at":"2026-08-21T18:40:52Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6651,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6650/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090866482933141504","occurred_at":"2026-08-21T18:19:50Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6650,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090849047588900864","occurred_at":"2026-08-21T17:10:33Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090849031038177280","occurred_at":"2026-08-21T17:10:29Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090837373561606144","occurred_at":"2026-08-21T16:23:45Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090837156841918464","occurred_at":"2026-08-21T16:23:18Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6647/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090836865526534144","occurred_at":"2026-08-21T16:22:00Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6647,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090836156110344192","occurred_at":"2026-08-21T16:19:20Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090833818511478784","occurred_at":"2026-08-21T16:10:02Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090833794218070016","occurred_at":"2026-08-21T16:09:56Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090804889495015424","occurred_at":"2026-08-21T14:15:05Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090803404556210176","occurred_at":"2026-08-21T14:09:11Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6647/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090794933425278976","occurred_at":"2026-08-21T13:35:19Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6647,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6643/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090790935381151744","occurred_at":"2026-08-21T13:19:38Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6643,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090784604393836544","occurred_at":"2026-08-21T12:54:29Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090780869554868224","occurred_at":"2026-08-21T12:39:38Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090773034322366464","occurred_at":"2026-08-21T12:08:30Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090768490683174912","occurred_at":"2026-08-21T11:50:27Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6646,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6643/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090757087574364160","occurred_at":"2026-08-21T11:05:08Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6643,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090753518062276608","occurred_at":"2026-08-21T10:50:57Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090725937179004928","occurred_at":"2026-08-21T09:00:54Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6625,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090725692609138688","occurred_at":"2026-08-21T09:00:23Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090717616074133504","occurred_at":"2026-08-21T08:28:17Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090717583954153472","occurred_at":"2026-08-21T08:28:10Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090711731759222784","occurred_at":"2026-08-21T08:04:32Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090693922316619776","occurred_at":"2026-08-21T06:54:08Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090688754837622784","occurred_at":"2026-08-21T06:33:36Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090683781416488960","occurred_at":"2026-08-21T06:13:51Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090677551943913472","occurred_at":"2026-08-21T05:49:05Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090660118717796352","occurred_at":"2026-08-21T04:39:49Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090659366838472704","occurred_at":"2026-08-21T04:36:40Z","signature_id":"8d39e883141cce57174faa8cb2ef7cb365b341ad5d50460a3e52f225fee04fd2","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657643948412928","occurred_at":"2026-08-21T04:29:59Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090644128227397632","occurred_at":"2026-08-21T03:36:17Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090643626764800000","occurred_at":"2026-08-21T03:34:17Z","signature_id":"1885b54e7935000de836bb799b144a9f3247eb31db63883364f7bb8dc63e0a64","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e4f1a1ea46940043a3767809b952d9be40a221815d3f81ceb038491c52599831","failure_pattern":"alert [svc] KubeDaemonSetMisScheduled fired","search_query":"alert [svc] KubeDaemonSetMisScheduled fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetMisScheduled does not fire","occurrences":44,"runs_affected":44,"failed_runs":44,"run_impact_percent":7.4957410562180575,"anchor_occurrences":44,"after_last_push_count":65,"trend_counts":[15,15,9,9,10,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":207,"prior_last_seen_at":"2026-08-21T21:25:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetMisScheduled does not fire","occurrences":44}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet velero/node-agent are running where they are not supposed to run.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T16:12:09Z\n  Ended:       2026-08-24T16:24:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.\nFiring 3:\n  State:       Resolved\n  Started:     2026-08-24T16:13:10Z\n  Ended:       2026-08-24T16:23:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.\nFiring 4:\n  State:       Resolved\n  Started:     2026-08-24T16:13:11Z\n  Ended:       2026-08-24T16:23:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet velero/node-agent are running where they are not supposed to run.","alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet velero/node-agent are running where they are not supposed to run.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet velero/node-agent are running where they are not supposed to run.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:50:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.","alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet velero/node-agent are running where they are not supposed to run.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"node-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"velero\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet velero/node-agent are running where they are not supposed to run.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetMisScheduled\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: 1 Pods of DaemonSet arobit/arobit-forwarder are running where they are not supposed to run."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"7caa0010541a7c4ff937ecc7f1700635aa06e8c5ef418d2f4d2c09799f427a78","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090913249020088320","occurred_at":"2026-08-21T21:25:40Z","signature_id":"d85d02d9cb6d2c75be2eab26be90eebd98525d6443b746d221016f89fc701fb5","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090912033770835968","occurred_at":"2026-08-21T21:20:50Z","signature_id":"1f5d187ce251780f799514cf1123320091fcff9c835d9bfa0f2e34e82c80dc24","pr_number":6651,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894675127832576","occurred_at":"2026-08-21T20:11:52Z","signature_id":"fd26538be5506281b89a0f2bdc1f564962744e634a6f244b5fb748dff210306f","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090849031038177280","occurred_at":"2026-08-21T17:10:29Z","signature_id":"3d7a57b08e70dfcb08eef19bdaef6376a8198c105bd85ba8dfaa49e85fcd4b23","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"823f5b2aa8486a7acbe1e124e698dd28e6761a0aee8a4da1641d137d9f965857","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090676319984553984","occurred_at":"2026-08-21T05:44:12Z","signature_id":"775472cbd3ba6274119d9bc4ff2a717c920fd383df136f13e36acce94c0bdfb6","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090660118717796352","occurred_at":"2026-08-21T04:39:49Z","signature_id":"08ee6d6fe4f72157b7e60bdf3099792d877119ac9d5be1390164d8d37baad484","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657643948412928","occurred_at":"2026-08-21T04:29:59Z","signature_id":"67a7460dd8f9b8fd2618e9491341d8f7e95c65de245a49249b7bdd6d050e5828","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"17225edf7352999500f539996fb1e01462e7b62a705f105c43a815c2c5405f67","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090612430223708160","occurred_at":"2026-08-21T01:30:11Z","signature_id":"e27081bfeef47cc4451336118ca5c4a1a5e79e92b0462a62edd92bb442b7c0e3","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090587456947621888","occurred_at":"2026-08-20T23:51:05Z","signature_id":"85ae2a4c300dcd3c7cd534d580abccbc52517c8a99dcb6c729d5062e671c8397","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"9fe5db5cb089dd418c555924d53d7186d613f25ae9f6c4b559ff7d0fec98563e","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090497381098328064","occurred_at":"2026-08-20T17:53:09Z","signature_id":"c6814be68413071dff1b4160fa6464cab99366cb0d562af5e197e45927105455","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"153158a58298050b340d840baeefe1849b5a5f210965465e880847adee541906","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090473213871001600","occurred_at":"2026-08-20T16:17:01Z","signature_id":"70c2460bf40893e6a3319184633a62cd5392e1fa1da4d377e367e9688a26623d","pr_number":6621,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090443849187987456","occurred_at":"2026-08-20T14:20:26Z","signature_id":"5fffed194c8b221512b1de9118320c058d429ab88d6a98085b57636306ebda90","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6589/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090373215313465344","occurred_at":"2026-08-20T09:39:46Z","signature_id":"a59681b8fb4c8d535dfc9ebe1ce2f0bffe47d780f475ed48cd4e44fd1eaf7ec3","pr_number":6589,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"955bfb75d761e55c3947d67a37a34c6f865e93d558669c6e553c2c55fff53b0a","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"12c751faedf8b07d89a7b665d957a08a566082aedba6ef103a3ae86e8b16bbe1","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"61bb80579bb3f03389e124f657ef9a4580750698aaec8f6d4a5cb42fd9c5a58e","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090126361980047360","occurred_at":"2026-08-19T17:18:51Z","signature_id":"6d0c2cf5e46ab65efd59eafc90530d6afea5da68b455b1c9a1ed74acdea6ab01","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090119905717260288","occurred_at":"2026-08-19T16:53:12Z","signature_id":"f33fa63e75243f8f6c0662e8c432ef5ef1d02d6f6d8078b95a60a5b86a1e252a","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090111171486027776","occurred_at":"2026-08-19T16:18:30Z","signature_id":"ec22af5eee9a5decc3de2f2d105970ef83ec4375005a081c6224c7b8c664f906","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090087196852752384","occurred_at":"2026-08-19T14:43:14Z","signature_id":"ab975ddc2ec3e21b23c47ba3911f432624317d5e2ab06dabdb76af0d67467eaf","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090087137016811520","occurred_at":"2026-08-19T14:42:58Z","signature_id":"f4dc165345eaa96f69c587c27e604b2d16e6b78b53c717ec949a8df90b91519f","pr_number":6553,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"84e0dbe296baf5af2dc285e38ecf1486e8c1d78306f03ff21441b3348759b746","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090072497390096384","occurred_at":"2026-08-19T13:44:49Z","signature_id":"6ebffb8dbc25915240a6cfbfe96d94fbca4a5dc38b4159455c3c48ec63042680","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090035242617278464","occurred_at":"2026-08-19T11:16:37Z","signature_id":"fe495db1d221c4a8c124437bf73ed3f249a488629d3f37aad664611de5f9690b","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089829163719462912","occurred_at":"2026-08-18T21:37:54Z","signature_id":"a54ea677fad8293e78dcc1ef9f80b44d8c15c2abc474aa54eba20a50c1ea3c8d","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089809279165927424","occurred_at":"2026-08-18T20:18:53Z","signature_id":"c708921a6df4062aaf859a1103afd858a650376bc973959017a141cd463b90cd","pr_number":6562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089768451781431296","occurred_at":"2026-08-18T17:36:39Z","signature_id":"1727b1de232034500e942e1aab8cf944f45c7fa74ffb1aa291f297619f0a1312","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089759198756737024","occurred_at":"2026-08-18T16:59:53Z","signature_id":"35f18f92ec803e8bb8165cc1030ef5fa4c09cb51389049f9ef3db2d90820f50e","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089757851600818176","occurred_at":"2026-08-18T16:54:32Z","signature_id":"e71c9ce91669fd8324e6f8bf8e85719a15413673f22953cb779bf7cf89e26ad7","pr_number":5880,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089753182908452864","occurred_at":"2026-08-18T16:35:55Z","signature_id":"ae623ea2ffa6a48e1a3a45f8e082723b1ec87ebc731e4573907e51dab7bf936a","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"5b3ad207eeaa06b6a21b807b9ad9211d696dcd10f8acd529f76dd5d56ca2d7e6","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089723779814002688","occurred_at":"2026-08-18T14:39:04Z","signature_id":"83eac450792b36206d3c6202eced7bce13276fde8c391ebb12ef623d7011eee9","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089713360550498304","occurred_at":"2026-08-18T13:57:44Z","signature_id":"c27f3a1f94ad861cc54b2dcc6b157266668bc7c3031da207e301dacdddebd8e9","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708056525934592","occurred_at":"2026-08-18T13:36:10Z","signature_id":"055f29daf958cbcd7f573972bf0ce1d01d40d505e93776a99fc9d6d5436ab076","pr_number":6585,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6426/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089706907987087360","occurred_at":"2026-08-18T13:32:06Z","signature_id":"02e9cff26bdcfca022ec9ac0bc5277b7e0b4011f41bb0c2b5ad304e6f8afb8c4","pr_number":6426,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089686738145382400","occurred_at":"2026-08-18T12:11:57Z","signature_id":"37f7fdbb6247ec3493c5719c2245cec66ba056895ab505e0669074f9c858d1cb","pr_number":5880,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"f65287cc9b68660910eadc1a397a9b7f25f21ec5b634e7b2aa16db875704d032","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089665224855523328","occurred_at":"2026-08-18T10:46:28Z","signature_id":"eb0adefaa868f66bf4cc35f2787315bdab6f3b89a298b6844e688c95dbdbb3be","pr_number":6573,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089651610870353920","occurred_at":"2026-08-18T09:52:22Z","signature_id":"a01b92da54e4f1c7cf86b585f80dfcf0f2b9b20a327284d5b7bed9ec3fbd9808","pr_number":6562,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"ea1b434856e874a026ae1b29ba51ae25f0803da2978c85901f04879bdcc66037","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":41,"runs_affected":13,"failed_runs":13,"run_impact_percent":2.2146507666098807,"anchor_occurrences":41,"after_last_push_count":108,"also_in":["PROD","STG"],"trend_counts":[2,3,2,0,27,5,0,4],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":148,"prior_last_seen_at":"2026-08-22T00:26:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:109]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000da71c0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-wqnnn79bk4qc\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e635655f-c9ac-44e1-8232-3550a4c20673\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e635655f-c9ac-44e1-8232-3550a4c20673\",\n      \"name\": \"e635655f-c9ac-44e1-8232-3550a4c20673\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:28:57.457434277Z\",\n      \"endTime\": \"2026-08-24T21:48:06.423004565Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:109]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000da71c0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-wqnnn79bk4qc\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e635655f-c9ac-44e1-8232-3550a4c20673\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e635655f-c9ac-44e1-8232-3550a4c20673\",\n      \"name\": \"e635655f-c9ac-44e1-8232-3550a4c20673\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:28:57.457434277Z\",\n      \"endTime\": \"2026-08-24T21:48:06.423004565Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:109]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa81a0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-jhvt8ptm7fw8\" to finish creating: GET https://rp.j8539776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/12485de2-db77-439f-89df-5bacb5d218b2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/12485de2-db77-439f-89df-5bacb5d218b2\",\n      \"name\": \"12485de2-db77-439f-89df-5bacb5d218b2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:41:38.67903036Z\",\n      \"endTime\": \"2026-08-24T16:00:39.109568382Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:109]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa81a0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-jhvt8ptm7fw8\" to finish creating: GET https://rp.j8539776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/12485de2-db77-439f-89df-5bacb5d218b2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/12485de2-db77-439f-89df-5bacb5d218b2\",\n      \"name\": \"12485de2-db77-439f-89df-5bacb5d218b2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:41:38.67903036Z\",\n      \"endTime\": \"2026-08-24T16:00:39.109568382Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e86920\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-7vpnvrbqnldf\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73cc6203-30af-4c53-9024-398221683c83\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73cc6203-30af-4c53-9024-398221683c83\",\n      \"name\": \"73cc6203-30af-4c53-9024-398221683c83\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:17:58.555796542Z\",\n      \"endTime\": \"2026-08-24T15:37:08.01693151Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e86920\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-7vpnvrbqnldf\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73cc6203-30af-4c53-9024-398221683c83\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/73cc6203-30af-4c53-9024-398221683c83\",\n      \"name\": \"73cc6203-30af-4c53-9024-398221683c83\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:17:58.555796542Z\",\n      \"endTime\": \"2026-08-24T15:37:08.01693151Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"409a88ec07ceec5d40d5ae23f780d5c74cad626824811548eedaa155d1eba475","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","signature_id":"3605097c99e7a1623e2e3dd91c5b9641833e016d1c0c516d9b0275085f65863f","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"4de409ed2fffcc46ddb557100c947bf3b037028f8c648ac70391763c58835df0","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895460682797056","occurred_at":"2026-08-24T14:28:36Z","signature_id":"2cc87e67c265ba4f74a9a313e8f38a6f54437455cae5243f5e1f2cc3556cdfc9","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090958673797976064","occurred_at":"2026-08-22T00:26:10Z","signature_id":"32e3e639e79da48278f94d8391d74ce4cb85a363903f3ef9711f2b4a60031ae5","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090958673797976064","occurred_at":"2026-08-22T00:26:10Z","signature_id":"ef641fb77f87b6886bc522fb57a3da8fe06dbc7d26494080a421abb9ad752057","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090958470403592192","occurred_at":"2026-08-22T00:25:22Z","signature_id":"a064f13405d704dd956551e506d55eae97dc3afdd6ceed198a730a6dc40f8e9a","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090954745601593344","occurred_at":"2026-08-22T00:10:34Z","signature_id":"90e64bea2b5cd9fda39e4d27f36261d3fc85d5ff17672231eeb0e009c5e68bd7","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090954745601593344","occurred_at":"2026-08-22T00:10:34Z","signature_id":"abb07177c1d013f32243889baee16622a26170afeea839f5d47b0654ffbf9ac2","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"0349ddcd04864267fea2a40c6ffd0f5389f34b53149c3626152112cae34b7449","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"0551f4776b518df224c09b5adbdb96739a175249d48cb4ac3fac68e653ba2784","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"0c33d2cf14048080901688f04f7f168fea8a7a6a51e4092f2878897e923597fe","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"0e1a706139de5aa4816eca677714434847c2c72fbb8ca3cdf11469904c0b488e","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"104b8359ad40ff28797d2cc44d09f2edb8fa2cea7a9596a7bde475000ff6418a","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"1e1bf8d1b89ceea99d469469baddd96086177c0cdc9195d3a71cbb63354e5d71","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"22969b4eb4fa35e9e8b87425aca1b945193cd797a33ff2424068183b2e6ab9d5","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"2b14f6adbf7b7b91812954e1a14265de8a301e9d5fe741ae9b10e7b9bb6acf09","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"2e21fe98b5b1c02d97edb490b4df36950ff6461c0fb4f46deff583fbba4d3c69","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"4771c1f60aef391df6d58cb19d51e18d1459e9cdd584c483d829a743260e5437","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"5765f0a35ed890055f1423af6fb083204d4880be33ff573010e5ce36cde4c153","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"578e27c8f27eed3fe0ee1c891072f72be7bb6bef2e19cd87df7a2fc2e6f926e7","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"5cacea6c38638a09cbc35d06a63928290b6a37ac1805cf72182801f515e5b20e","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"5dafea2e96443aebcae093f90aba801deaac09c85776b69fee7e3e9f98f2ae4e","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"7f15a6d4df92119c16c31a6389fc33139cb6399bdee2ed13425acfdb032fa99c","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"7fd612d767c2ea3564f764b73b42a11ca9eea2b84ecbe772fd21a9c566dfae7b","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"89aba46a0c865e129eed75a6b31c0b3bc5dc7f077becd5cb01904d6e859b0c22","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"8a4cfe4d036a329b0fd9efe82e95b5a8589a1ed16beb75cbefa82f94fc66963c","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"957b662b32c881f5058300aba160316b07ec41cf6cde6fb9d85df6dea5cd5603","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"9a63130286fb0ef860c5f8085bc0379c1e9f1367550fb2fd9bb136a90e890736","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"a034d5a2d2675bf1f04eb0cf418b007ad1dff484ef6e4352e995b9ee9d000949","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"cf1f5ad07e8fcc4b5ad4eabc673ebe14ec2d966a56dd2f43bf7d01e12685a642","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"d21bd18a17ffb4f907acdd9f461d092ef8af245bb3b161c6ca73eb7c3b3c5126","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"d5cfde077190f3c18ba68ca0d18e114870bd9d384119e2b5f40b6488f746be3e","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"dad7476bfcf96373dfa42e63321486c9e8ee70265a767ae521fcda72aef7cd4e","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"e28e93d7094aad43c209ccd557ce0e6a208211122c2f3b778170a8496e2a9423","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"edcef64fd444b5b8f7c061e41f4f70744ef8ed0ca26c4e9d7a061b5ef6aa7762","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6596/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090162950290542592","occurred_at":"2026-08-19T19:44:15Z","signature_id":"f2bf06ca84a5627688967789c6d03c70cf37e834531750d81991aaaf88fece21","pr_number":6596,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"ffdb59f5b7f9768099566982a694648158846401c53b2afb3f022b9d904fa4e5","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089740517767122944","occurred_at":"2026-08-18T15:45:39Z","signature_id":"0c0da5bfecae80945d7476d8ab8d6e08bd0ddd24140b18dd124ec93ce4f6befe","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089723779814002688","occurred_at":"2026-08-18T14:39:04Z","signature_id":"da007b9333be80c8fde66889233cd45036cdb566cd3c0b6090a58d2418e4ded2","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089679732839813120","occurred_at":"2026-08-18T11:44:07Z","signature_id":"e4e1b1e0185174445aa84f90f1d2e01d6bfd60e1df2a492cc5e9127507dfe4db","pr_number":6553,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":39,"runs_affected":39,"failed_runs":39,"run_impact_percent":6.6439522998296425,"anchor_occurrences":39,"after_last_push_count":48,"trend_counts":[10,16,6,8,5,3,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":190,"prior_last_seen_at":"2026-08-22T08:48:46Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":39}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T18:15:21Z\n  Ended:       2026-08-24T18:36:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-22T10:06:50Z\n  Ended:       2026-08-22T10:14:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3442688-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.129:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3442688/providers/microsoft.monitor/accounts/hcps-j3442688\", namespace=\"ocm-arohcpci01-2sbhtcj4djmllishlrj1ubeicnu0g5pf-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-hp77b\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sbhtcj4djmllishlrj1ubeicnu0g5pf-sre-hcp-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 16 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-22T06:09:36Z\n  Ended:       2026-08-22T06:19:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/hcps-j8029824\", namespace=\"ocm-arohcpci01-2sbeeil0i6mt7oboc04nhglhna7qqum0-k2t7b3m6u6j2z9c\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sbeeil0i6mt7oboc04nhglhna7qqum0-k2t7b3m6u6j2z9c/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-22T06:10:37Z\n  Ended:       2026-08-22T06:17:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/hcps-j8029824\", namespace=\"ocm-arohcpci01-2sbeepf04jq44auu7rr8i1ok2dcjhtlu-h9f7b5h9h4k7x9r\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sbeepf04jq44auu7rr8i1ok2dcjhtlu-h9f7b5h9h4k7x9r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-08-22T06:10:37Z\n  Ended:       2026-08-22T06:19:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/hcps-j8029824\", namespace=\"ocm-arohcpci01-2sbeepf04jq44auu7rr8i1ok2dcjhtlu-h9f7b5h9h4k7x9r\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sbeepf04jq44auu7rr8i1ok2dcjhtlu-h9f7b5h9h4k7x9r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-08-22T06:10:37Z\n  Ended:       2026-08-22T06:16:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/hcps-j8029824\", namespace=\"ocm-arohcpci01-2sbeekbk6283ad3oksf8bsv67jh0l3j8-cilium-cl\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sbeekbk6283ad3oksf8bsv67jh0l3j8-cilium-cl/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-08-22T06:11:38Z\n  Ended:       2026-08-22T06:31:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/hcps-j8029824\", namespace=\"ocm-arohcpci01-2sbeee9sknhjmru28j861qqr85i8o22i-q0s4s8a3f6d6o5k\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sbeee9sknhjmru28j861qqr85i8o22i-q0s4s8a3f6d6o5k/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-08-22T06:11:38Z\n  Ended:       2026-08-22T06:19:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/hcps-j8029824\", namespace=\"ocm-arohcpci01-2sbeek9brcripht70j68qvmcbnvoclaj-d7k7d4n7p7p2q4v\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sbeek9brcripht70j68qvmcbnvoclaj-d7k7d4n7p7p2q4v/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-08-22T06:11:38Z\n  Ended:       2026-08-22T06:19:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/hcps-j8029824\", namespace=\"ocm-arohcpci01-2sbeeir6uue5hi968m1tal3vd4g0p4tk-r4y1m1u1q4k5j4w\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sbeeir6uue5hi968m1tal3vd4g0p4tk-r4y1m1u1q4k5j4w/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-08-22T06:11:38Z\n  Ended:       2026-08-22T06:17:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/hcps-j8029824\", namespace=\"ocm-arohcpci01-2sbeehpv4kuqpssgmdcbocp5oq1soheu-v6i5c0r2t6w5h2g\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sbeehpv4kuqpssgmdcbocp5oq1soheu-v6i5c0r2t6w5h2g/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-08-22T06:11:38Z\n  Ended:       2026-08-22T06:17:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/hcps-j8029824\", namespace=\"ocm-arohcpci01-2sbeek9brcripht70j68qvmcbnvoclaj-d7k7d4n7p7p2q4v\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sbeek9brcripht70j68qvmcbnvoclaj-d7k7d4n7p7p2q4v/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-08-22T06:11:38Z\n  Ended:       2026-08-22T06:17:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/hcps-j8029824\", namespace=\"ocm-arohcpci01-2sbeempoqgao8mpiu344ftttlggj568f-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sbeempoqgao8mpiu344ftttlggj568f-private-ingress/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-08-22T06:12:37Z\n  Ended:       2026-08-22T06:19:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/hcps-j8029824\", namespace=\"ocm-arohcpci01-2sbeehpv4kuqpssgmdcbocp5oq1soheu-v6i5c0r2t6w5h2g\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sbeehpv4kuqpssgmdcbocp5oq1soheu-v6i5c0r2t6w5h2g/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-08-22T06:12:37Z\n  Ended:       2026-08-22T06:19:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/hcps-j8029824\", namespace=\"ocm-arohcpci01-2sbeempoqgao8mpiu344ftttlggj568f-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sbeempoqgao8mpiu344ftttlggj568f-private-ingress/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-08-22T06:12:37Z\n  Ended:       2026-08-22T06:31:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/hcps-j8029824\", namespace=\"ocm-arohcpci01-2sbeee9sknhjmru28j861qqr85i8o22i-q0s4s8a3f6d6o5k\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sbeee9sknhjmru28j861qqr85i8o22i-q0s4s8a3f6d6o5k/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-08-22T06:12:37Z\n  Ended:       2026-08-22T06:19:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/hcps-j8029824\", namespace=\"ocm-arohcpci01-2sbeenmev23bl18jkrt2lmlsssg9k4gb-j0m5o8b7c0z0v9w\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sbeenmev23bl18jkrt2lmlsssg9k4gb-j0m5o8b7c0z0v9w/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-08-22T06:13:36Z\n  Ended:       2026-08-22T06:21:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/hcps-j8029824\", namespace=\"ocm-arohcpci01-2sbefdpdepfvfi4jatmrhbejsk2d6fbr-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sbefdpdepfvfi4jatmrhbejsk2d6fbr-cilium-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-08-22T06:13:36Z\n  Ended:       2026-08-22T06:19:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/hcps-j8029824\", namespace=\"ocm-arohcpci01-2sbeep8cvaflqvdh2ollk224hmbcdpga-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sbeep8cvaflqvdh2ollk224hmbcdpga-v20260901/capi-provider has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"21c20c66c7c9aa8660ddbc1f0e9827cf7f85ba5f2e371e49fb9660d2202ea0a2","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091085161763442688","occurred_at":"2026-08-22T08:48:46Z","signature_id":"ea45d80556c6df1b7dcdc93568199771da13a2e986e58392825c14f49b9dbf7a","pr_number":6640,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"e455d9b7b038a17e57cc035ca9c9c5f5a28eaf76b1c22688c29b969f7b31ee75","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090960143322714112","occurred_at":"2026-08-22T00:32:00Z","signature_id":"704ddf6adff4d039aa03923c06032bc68e7d6d5f5f219a74df97dc060fd8fc0b","pr_number":6639,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090913249020088320","occurred_at":"2026-08-21T21:25:40Z","signature_id":"351d71055a17078b68a005f194223b4a24156deb912d46427e913acd0aeb8777","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090912033770835968","occurred_at":"2026-08-21T21:20:50Z","signature_id":"2e25fc232dcb33241fcd0cb57973dad8d90728d5fe6dcfb8afc166949d3b12a7","pr_number":6651,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"0c7f8c5ea85d16164325cbe2e383117ecacd076d46ac3359cbd59f601532eb44","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090676319984553984","occurred_at":"2026-08-21T05:44:12Z","signature_id":"72701a355bc04323206fcd1c0bd2b36b578f4d899899554804fcd06228542320","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"b30701a5cf5cf655007b0c0e5396a33379506a4b879c38d39601b83fa7527486","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"f5e5986bcef7a7e209bf0fed58426b925ed47dfd3408962879b99b77392036c5","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"03d4ae8262f1770e01a1d35260bf8e34f79b37a5aaac55583e71924d479302f8","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090497381098328064","occurred_at":"2026-08-20T17:53:09Z","signature_id":"5db4b0d2292280947c2278091bb4c40b2d5b8ca5b79b2d7c07f2c2e0859a0850","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"677320e48c2626b64b19de7b052b3fb050b9e5acc1ec61005b4e06f51178e853","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090465920358027264","occurred_at":"2026-08-20T15:48:08Z","signature_id":"be8f5ddd90fd2dc0cabb580d1ffa28061209566ae2296abdc713b4d8b56c7b97","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"3754690aedcadcebdaafefd53de15e8476013f726ec2d2b70ec0ae536dea3439","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"fd0ea1182bc283fb8ca2a0c6dd6b26ae5bd1e3dad84a163a320485814a6ea5c0","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"ee648d4d3cb0ce098a24d4ad4a68319d02795b14bc529474fdb13e67830298d2","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090206132864815104","occurred_at":"2026-08-19T22:35:50Z","signature_id":"e4f5c665fa9bc4222d3c1be5d216af10b1bf7853150a7490188bfc4942188881","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"c63de78f05317712503aa77f111d1143e526f4ee67eb10bdd44087621b074be2","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090133175878553600","occurred_at":"2026-08-19T17:45:52Z","signature_id":"7a0e5fe89e8c70bbb67c4e405e89758b4a8d2fdfa426a3e6b132bc93a3b5a5ec","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090126361980047360","occurred_at":"2026-08-19T17:18:51Z","signature_id":"f28a6eded26a41ba3b81af27f2c8ab7a2fb450ba34046088f326e7c899b05e6c","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"48bd353f139412a8282a326a12d2c8de87fc5c59236878c955792c79218a1005","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090072497390096384","occurred_at":"2026-08-19T13:44:49Z","signature_id":"48394d74a095ad1f33da1153c255bd41f1fb141941e6fd8809cbfbe09968b204","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089850706507862016","occurred_at":"2026-08-18T23:03:30Z","signature_id":"57360f9e4acdcc4f46cce2742c20ac2d66a9e55b2ee75a22a9bbcd7f37fcb97a","pr_number":6132,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089829163719462912","occurred_at":"2026-08-18T21:37:54Z","signature_id":"d3a7d69191803a3a5129384fce21c09599964893aa2594ff864c2b771f3a9c2b","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089809279165927424","occurred_at":"2026-08-18T20:18:53Z","signature_id":"bd963981180158b318c22465ef2ca3bc571c6086c10fb2e5a71344017b2bdb5a","pr_number":6562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089757851600818176","occurred_at":"2026-08-18T16:54:32Z","signature_id":"97fea652f434ce2deede14f8e8ba046ac675a3cc565e6ff056d73227bd4a2721","pr_number":5880,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089754858566455296","occurred_at":"2026-08-18T16:42:38Z","signature_id":"124d4158b659a8aad185f28fe269c2b9fab249e56eddf75babae9ddda5aa9c12","pr_number":6553,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089753382901256192","occurred_at":"2026-08-18T16:36:46Z","signature_id":"667e572029c1b939da15eb18e255937bd76c4599a4ee167cb93306ee1f226846","pr_number":5900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089753182908452864","occurred_at":"2026-08-18T16:35:55Z","signature_id":"5564bf68099460118c458c69295fda59666089882acf920162a9faacdf2e80e5","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"c693bae98b7e3f7c1c3c60248e6ad57927cf4e1d862db0b9e5a1867ce56da0c3","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089723779814002688","occurred_at":"2026-08-18T14:39:04Z","signature_id":"7652a82abec4375d25fa569eda0d0331568dde95fe5ebf76961592e9aa77b022","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089713360550498304","occurred_at":"2026-08-18T13:57:44Z","signature_id":"6755e9d2b4beec63aea616d5139d72f9eb3c40fa57e7f81d5f78b20177a2aea2","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708056525934592","occurred_at":"2026-08-18T13:36:10Z","signature_id":"1f4be917bf8e978ce53a114f5bfdd2403bce6c65e3af0c1af5a7b7eb16a51dee","pr_number":6585,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6426/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089706907987087360","occurred_at":"2026-08-18T13:32:06Z","signature_id":"0480efd0909f44c6e126ed6e1aceef909be3e7927907cdd84719ba6636c1fcd0","pr_number":6426,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089686738145382400","occurred_at":"2026-08-18T12:11:57Z","signature_id":"e6883ba127bc15f0d8f70fc74eb1c4b413ec106a8fe720fd43002bc29f2b5c5a","pr_number":5880,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"ab31f31719e4cb1fe31ba46786146e0b33f15eedfa4852c0de581ac73f7d57d3","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089665224855523328","occurred_at":"2026-08-18T10:46:28Z","signature_id":"55751496605e8b17f98775832431fb1687d40e932a58e54988dd7b563f6167db","pr_number":6573,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089644114403397632","occurred_at":"2026-08-18T09:22:35Z","signature_id":"b80dade67bad4a870538242ed2511bcde5f7abb35e1d86da95e914554a9494d6","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d6c6b690ca60846dd98e464313d6a9672b9abd8744604edd915627eb68203f9d","failure_pattern":"alert [svc] KubeNodeNotReady fired","search_query":"alert [svc] KubeNodeNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeNotReady does not fire","occurrences":38,"runs_affected":38,"failed_runs":38,"run_impact_percent":6.473594548551959,"anchor_occurrences":38,"after_last_push_count":52,"trend_counts":[9,12,10,8,8,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":182,"prior_last_seen_at":"2026-08-21T21:25:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeNotReady does not fire","occurrences":38}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T23:03:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeNotReady\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"prometheus\", node=\"aks-userswft1-18345013-vmss000002\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n  Description: aks-userswft1-18345013-vmss000002 has been unready for more than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T23:04:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeNotReady\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"prometheus\", node=\"aks-userswft1-18345013-vmss000002\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n  Description: aks-userswft1-18345013-vmss000002 has been unready for more than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T23:03:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeNotReady\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"prometheus\", node=\"aks-userswft1-16379006-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n  Description: aks-userswft1-16379006-vmss000000 has been unready for more than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T23:03:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeNotReady\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"prometheus\", node=\"aks-userswft1-16379006-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n  Description: aks-userswft1-16379006-vmss000000 has been unready for more than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:02:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeNotReady\", cluster=\"ci01-j7832576-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.184:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7832576/providers/microsoft.monitor/accounts/services-j7832576\", namespace=\"prometheus\", node=\"aks-userswft1-31866601-vmss000003\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-rfcdk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n  Description: aks-userswft1-31866601-vmss000003 has been unready for more than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:03:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeNotReady\", cluster=\"ci01-j7832576-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Ready\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.184:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7832576/providers/microsoft.monitor/accounts/services-j7832576\", namespace=\"prometheus\", node=\"aks-userswft1-31866601-vmss000003\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-rfcdk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"true\"\n  Description: aks-userswft1-31866601-vmss000003 has been unready for more than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090913249020088320","occurred_at":"2026-08-21T21:25:40Z","signature_id":"2d49350805ce859d57a8cd273a371aaa89e52a74ab0ec67b49ebbe6af5b52d3c","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090912033770835968","occurred_at":"2026-08-21T21:20:50Z","signature_id":"1dd5d3d158481f1b778faeeafc16d713505247bf52c9b2150dab32d0157936a2","pr_number":6651,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894675127832576","occurred_at":"2026-08-21T20:11:52Z","signature_id":"f426881115951519f9389af0904a3bed97befe021b18b71006baa4ac30465558","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"0d42f8d6d6935dd62ed8dbeb9c7e3e36edece6c01ae7f4ad36b523de5c16f59f","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090676319984553984","occurred_at":"2026-08-21T05:44:12Z","signature_id":"391e72b3f41ca31b44cc2156ee2d00504773a97d0b9001121531eb40acc62c51","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657643948412928","occurred_at":"2026-08-21T04:29:59Z","signature_id":"50ed525a4e9c2c924b6e4c4dbd838ce38633b6d5809af050d67f8522a0857269","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"774245db4660623d4f87e4a9aab111e2ef5d0f1bbb16244e0f417dcd42b5cdc9","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090643626764800000","occurred_at":"2026-08-21T03:34:17Z","signature_id":"b1383ba37c59ad31a6376fa07ce2d216fd51357d35e203592cce52c4214a960d","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090587456947621888","occurred_at":"2026-08-20T23:51:05Z","signature_id":"03b8eeb862b3000376dbbf65c43e29d1d044f9cf45378cd0848c6ab5cae80fd7","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"16b785165c7398914a8e20defdd01e667418ba06bfc083feec0c48b05787871c","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"161ba18c834f66ee1dc848538f0020fb2526c47e9571c94c76bbd2cd532c5ac3","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090473213871001600","occurred_at":"2026-08-20T16:17:01Z","signature_id":"1f17d318d94c5fe2c71f0d60b8bf5c0656f1cbb6ed461985c8328a2136615e75","pr_number":6621,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090443849187987456","occurred_at":"2026-08-20T14:20:26Z","signature_id":"53acdaa3429f7031060b3d1d59848e2a333fb88cabb6b0058770adb211218484","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090419688172425216","occurred_at":"2026-08-20T12:44:10Z","signature_id":"fb0987f312f9186fe0c7047695760c4065c9de8f5c4a5a7d016ab884637adb85","pr_number":6617,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"471d7ef1f6787056c1e53c9f3e004ce11ca87bc63edfd5c891057ce365dfff2d","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"a82ec01c8cccff1f4eccc628f23282745da5fc11afcb1c7a3043656d526e3af7","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090180258316685312","occurred_at":"2026-08-19T20:53:01Z","signature_id":"78c9f7739387864e1541324ffece4d4ea39fa817ba903bfa58d7a29d048cf9f4","pr_number":6614,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"0b67643102a1580039188783547fb06811c61148f505d0ffead658e5b91844c1","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090126361980047360","occurred_at":"2026-08-19T17:18:51Z","signature_id":"830394846f5cac4afc09f57187503aef9e826ac2ac99258336feb5d1cb36bcb0","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090119905717260288","occurred_at":"2026-08-19T16:53:12Z","signature_id":"7ed39c4abba50d92c5c7504ba592834e0b1647a9a852214043e3fbe6989d14a2","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090111171486027776","occurred_at":"2026-08-19T16:18:30Z","signature_id":"3d5d2a5a3cc187abd2d61709b5ca5f2533906e15dc434ba2181c758948bd12bd","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"88965bd8d63b9b169b13dcd3437f55796c215100cbd854f4d3437c1cd42a0e11","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090072497390096384","occurred_at":"2026-08-19T13:44:49Z","signature_id":"8e3321d42d8aacd9fbb824acefdb20e6a55ee2552b45adeb84026dc9aaf03cf6","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090035242617278464","occurred_at":"2026-08-19T11:16:37Z","signature_id":"6ccf240cb2a00ffe7d808440e9b97f42f94c7c8b737432c23072b646299858c8","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090031884472946688","occurred_at":"2026-08-19T11:03:26Z","signature_id":"53dbfbe54fc3f1c917661518de4e9bc09c6d0a2092c07e440e67b2fe23bc8da1","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"9740659f3211c4e67e9eddf1451df47f3260c679950a37e9f027cb815969d67c","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089829163719462912","occurred_at":"2026-08-18T21:37:54Z","signature_id":"c0b6dc78e358ea9949c862537228f320613f24a3d750dd19f421475ad42c6085","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089809279165927424","occurred_at":"2026-08-18T20:18:53Z","signature_id":"75138a3a86ff5584a798c72464c419c2cfdc43fe61bfcfe8f54e206e8f25160b","pr_number":6562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089768451781431296","occurred_at":"2026-08-18T17:36:39Z","signature_id":"5f54b7254fdfc9daa97f54215149be9942f1e7bfd8bfbcc270ac8fca1720dcf0","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089757851600818176","occurred_at":"2026-08-18T16:54:32Z","signature_id":"3523d229a3a71daddebc28419b994bc144fb9369749cbecce5563dd25e41d776","pr_number":5880,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089753182908452864","occurred_at":"2026-08-18T16:35:55Z","signature_id":"2e2e1bd321ca065fa68ec0f1bcd495d3cbb40b6d27511114aace70d3ef8536a9","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089723779814002688","occurred_at":"2026-08-18T14:39:04Z","signature_id":"42935e7c58f129fdb23c00cfc2d6bda9861904cf6c6158b891f62ce5741dba0a","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089713360550498304","occurred_at":"2026-08-18T13:57:44Z","signature_id":"b95909d6e9296813710d4ae4d1f5e07ddb687e69cfaf3e1a1bbb925dc1372dcc","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708056525934592","occurred_at":"2026-08-18T13:36:10Z","signature_id":"94a9e298e316c5a32c7fe3da9b203474855617f13264611a1fae7c23ceac08a4","pr_number":6585,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6426/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089706907987087360","occurred_at":"2026-08-18T13:32:06Z","signature_id":"5fc3e1a14ab37ad2825a3fe61ad7543e90f6d0710d4c389a25d4c277ccfd1908","pr_number":6426,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089686738145382400","occurred_at":"2026-08-18T12:11:57Z","signature_id":"876d1cfe3b1d64f07ea30d01b019f2436ceff358a0b777d8a67fa00d04b6b176","pr_number":5880,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"d49a09bc31103f03cecede9eeb83fce826b45b95a124a8992f65cc7aaef8bae9","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089665224855523328","occurred_at":"2026-08-18T10:46:28Z","signature_id":"5ab26aa7b4b61a05022e605cbd02d3050215e237271441063ef2dbc53979815a","pr_number":6573,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6283a00e01badbe18e51a0c7936b9753eb69a0c088d2d52d5debd15b846d4697","failure_pattern":"query-basic: command failed: exit status 1","search_query":"query-basic: command failed: exit status 1","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":35,"runs_affected":35,"failed_runs":35,"run_impact_percent":5.9625212947189095,"anchor_occurrences":35,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,35],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":35,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":35}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000b66310\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[00:48:59.861]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:48:54.9335446Z);\\nlet _endTime = datetime(2026-08-25T00:48:54.9335452Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:48:59.8208131Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45\\\",\\r\\n            \\\"activityId\\\": \\\"3f29cef9-55f3-4e39-a44c-dc2745aeb329\\\",\\r\\n            \\\"subActivityId\\\": \\\"9dc2e9b4-43ec-49f2-b454-4f12f219c5db\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"9b9edaeb-e277-470d-a49a-ba5e0f233409\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45 ARID=3f29cef9-55f3-4e39-a44c-dc2745aeb329 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9b9edaeb-e277-470d-a49a-ba5e0f233409 \\u003e DN.FE.ExecuteQuery/9dc2e9b4-43ec-49f2-b454-4f12f219c5db)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:48:59.984]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:48:54.9335446Z);\\nlet _endTime = datetime(2026-08-25T00:48:54.9335452Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:48:59.8208131Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45\\\",\\r\\n            \\\"activityId\\\": \\\"3f29cef9-55f3-4e39-a44c-dc2745aeb329\\\",\\r\\n            \\\"subActivityId\\\": \\\"9dc2e9b4-43ec-49f2-b454-4f12f219c5db\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"9b9edaeb-e277-470d-a49a-ba5e0f233409\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45 ARID=3f29cef9-55f3-4e39-a44c-dc2745aeb329 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9b9edaeb-e277-470d-a49a-ba5e0f233409 \\u003e DN.FE.ExecuteQuery/9dc2e9b4-43ec-49f2-b454-4f12f219c5db)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[00:49:04.412]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:00.0668173Z);\\nlet _endTime = datetime(2026-08-25T00:49:00.0668177Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:49:04.3919639Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef\\\",\\r\\n            \\\"activityId\\\": \\\"590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea\\\",\\r\\n            \\\"subActivityId\\\": \\\"f16f10a7-70e1-48c7-9e9d-209207cfc1e8\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"69739f9c-88ef-46ee-93e1-65af7b0a2416\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef ARID=590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/69739f9c-88ef-46ee-93e1-65af7b0a2416 \\u003e DN.FE.ExecuteQuery/f16f10a7-70e1-48c7-9e9d-209207cfc1e8)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:49:04.464]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:00.0668173Z);\\nlet _endTime = datetime(2026-08-25T00:49:00.0668177Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:49:04.3919639Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef\\\",\\r\\n            \\\"activityId\\\": \\\"590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea\\\",\\r\\n            \\\"subActivityId\\\": \\\"f16f10a7-70e1-48c7-9e9d-209207cfc1e8\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"69739f9c-88ef-46ee-93e1-65af7b0a2416\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef ARID=590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/69739f9c-88ef-46ee-93e1-65af7b0a2416 \\u003e DN.FE.ExecuteQuery/f16f10a7-70e1-48c7-9e9d-209207cfc1e8)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[00:49:12.903]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:08.6625258Z);\\nlet _endTime = datetime(2026-08-25T00:49:08.6625262Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:49:12.8815919Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;a5c94555-e249-4511-a807-b8477933031f\\\",\\r\\n            \\\"activityId\\\": \\\"3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c\\\",\\r\\n            \\\"subActivityId\\\": \\\"af4bd91d-24d7-4768-a750-045fb74a6811\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a5c94555-e249-4511-a807-b8477933031f ARID=3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa \\u003e DN.FE.ExecuteQuery/af4bd91d-24d7-4768-a750-045fb74a6811)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:49:13.008]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:08.6625258Z);\\nlet _endTime = datetime(2026-08-25T00:49:08.6625262Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:49:12.8815919Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;a5c94555-e249-4511-a807-b8477933031f\\\",\\r\\n            \\\"activityId\\\": \\\"3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c\\\",\\r\\n            \\\"subActivityId\\\": \\\"af4bd91d-24d7-4768-a750-045fb74a6811\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a5c94555-e249-4511-a807-b8477933031f ARID=3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa \\u003e DN.FE.ExecuteQuery/af4bd91d-24d7-4768-a750-045fb74a6811)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.001s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000b66310\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[00:48:59.861]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:48:54.9335446Z);\\nlet _endTime = datetime(2026-08-25T00:48:54.9335452Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:48:59.8208131Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45\\\",\\r\\n            \\\"activityId\\\": \\\"3f29cef9-55f3-4e39-a44c-dc2745aeb329\\\",\\r\\n            \\\"subActivityId\\\": \\\"9dc2e9b4-43ec-49f2-b454-4f12f219c5db\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"9b9edaeb-e277-470d-a49a-ba5e0f233409\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45 ARID=3f29cef9-55f3-4e39-a44c-dc2745aeb329 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9b9edaeb-e277-470d-a49a-ba5e0f233409 \\u003e DN.FE.ExecuteQuery/9dc2e9b4-43ec-49f2-b454-4f12f219c5db)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:48:59.984]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:48:54.9335446Z);\\nlet _endTime = datetime(2026-08-25T00:48:54.9335452Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:48:59.8208131Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45\\\",\\r\\n            \\\"activityId\\\": \\\"3f29cef9-55f3-4e39-a44c-dc2745aeb329\\\",\\r\\n            \\\"subActivityId\\\": \\\"9dc2e9b4-43ec-49f2-b454-4f12f219c5db\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"9b9edaeb-e277-470d-a49a-ba5e0f233409\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;e378a004-3ace-4448-8d86-93b49cc20b45 ARID=3f29cef9-55f3-4e39-a44c-dc2745aeb329 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/a5037b89-f047-4fed-bfb0-6c9466286aa0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/9b9edaeb-e277-470d-a49a-ba5e0f233409 \\u003e DN.FE.ExecuteQuery/9dc2e9b4-43ec-49f2-b454-4f12f219c5db)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[00:49:04.412]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:00.0668173Z);\\nlet _endTime = datetime(2026-08-25T00:49:00.0668177Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:49:04.3919639Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef\\\",\\r\\n            \\\"activityId\\\": \\\"590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea\\\",\\r\\n            \\\"subActivityId\\\": \\\"f16f10a7-70e1-48c7-9e9d-209207cfc1e8\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"69739f9c-88ef-46ee-93e1-65af7b0a2416\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef ARID=590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/69739f9c-88ef-46ee-93e1-65af7b0a2416 \\u003e DN.FE.ExecuteQuery/f16f10a7-70e1-48c7-9e9d-209207cfc1e8)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:49:04.464]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:00.0668173Z);\\nlet _endTime = datetime(2026-08-25T00:49:00.0668177Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:49:04.3919639Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef\\\",\\r\\n            \\\"activityId\\\": \\\"590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea\\\",\\r\\n            \\\"subActivityId\\\": \\\"f16f10a7-70e1-48c7-9e9d-209207cfc1e8\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"69739f9c-88ef-46ee-93e1-65af7b0a2416\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;4c4f5480-a590-4f9a-a459-65dc9658f8ef ARID=590d8d9f-c432-4b6c-a57a-14f6cfbdb0ea \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/b66f5a6d-cdfd-418f-82f1-af8b0b7b5e03 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/69739f9c-88ef-46ee-93e1-65af7b0a2416 \\u003e DN.FE.ExecuteQuery/f16f10a7-70e1-48c7-9e9d-209207cfc1e8)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[00:49:12.903]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:08.6625258Z);\\nlet _endTime = datetime(2026-08-25T00:49:08.6625262Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:49:12.8815919Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;a5c94555-e249-4511-a807-b8477933031f\\\",\\r\\n            \\\"activityId\\\": \\\"3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c\\\",\\r\\n            \\\"subActivityId\\\": \\\"af4bd91d-24d7-4768-a750-045fb74a6811\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a5c94555-e249-4511-a807-b8477933031f ARID=3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa \\u003e DN.FE.ExecuteQuery/af4bd91d-24d7-4768-a750-045fb74a6811)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:49:13.008]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:49:08.6625258Z);\\nlet _endTime = datetime(2026-08-25T00:49:08.6625262Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-dh6t5tfmwl78'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:49:12.8815919Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;a5c94555-e249-4511-a807-b8477933031f\\\",\\r\\n            \\\"activityId\\\": \\\"3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c\\\",\\r\\n            \\\"subActivityId\\\": \\\"af4bd91d-24d7-4768-a750-045fb74a6811\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;a5c94555-e249-4511-a807-b8477933031f ARID=3a1a3ba5-3fc9-4a7a-bbc2-c2b684f8ab2c \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/58e55353-9773-463e-ab07-afa8f479d884 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/833ff0b4-2415-4cdd-8b37-d9d4e7e62eaa \\u003e DN.FE.ExecuteQuery/af4bd91d-24d7-4768-a750-045fb74a6811)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.009s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00083c410\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[00:23:58.357]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:55.0351655Z);\\nlet _endTime = datetime(2026-08-25T00:23:55.0351661Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:23:58.3330999Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b\\\",\\r\\n            \\\"activityId\\\": \\\"442abb20-5490-45f5-ba70-3bca3bed0639\\\",\\r\\n            \\\"subActivityId\\\": \\\"b9132e8a-1c55-4f09-8982-f344852f0eee\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"99a04993-dace-423f-a879-2fbeee2a7b84\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b ARID=442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.Http.CallContext/442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.ExecuteQuery/99a04993-dace-423f-a879-2fbeee2a7b84 \\u003e DN.FE.ExecuteQuery/b9132e8a-1c55-4f09-8982-f344852f0eee)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:23:58.411]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:55.0351655Z);\\nlet _endTime = datetime(2026-08-25T00:23:55.0351661Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:23:58.3330999Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b\\\",\\r\\n            \\\"activityId\\\": \\\"442abb20-5490-45f5-ba70-3bca3bed0639\\\",\\r\\n            \\\"subActivityId\\\": \\\"b9132e8a-1c55-4f09-8982-f344852f0eee\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"99a04993-dace-423f-a879-2fbeee2a7b84\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b ARID=442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.Http.CallContext/442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.ExecuteQuery/99a04993-dace-423f-a879-2fbeee2a7b84 \\u003e DN.FE.ExecuteQuery/b9132e8a-1c55-4f09-8982-f344852f0eee)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[00:24:01.787]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:58.4821435Z);\\nlet _endTime = datetime(2026-08-25T00:23:58.4821442Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:24:01.7618170Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5\\\",\\r\\n            \\\"activityId\\\": \\\"d2adae1e-7609-4016-b166-5a8ff8be55d8\\\",\\r\\n            \\\"subActivityId\\\": \\\"7c0e685c-e2d3-436f-89bc-9d9fa3141b61\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"8cf560a3-f257-4d9d-900b-aaf46185fe8b\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5 ARID=d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.Http.CallContext/d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.ExecuteQuery/8cf560a3-f257-4d9d-900b-aaf46185fe8b \\u003e DN.FE.ExecuteQuery/7c0e685c-e2d3-436f-89bc-9d9fa3141b61)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:24:01.845]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:58.4821435Z);\\nlet _endTime = datetime(2026-08-25T00:23:58.4821442Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:24:01.7618170Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5\\\",\\r\\n            \\\"activityId\\\": \\\"d2adae1e-7609-4016-b166-5a8ff8be55d8\\\",\\r\\n            \\\"subActivityId\\\": \\\"7c0e685c-e2d3-436f-89bc-9d9fa3141b61\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"8cf560a3-f257-4d9d-900b-aaf46185fe8b\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5 ARID=d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.Http.CallContext/d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.ExecuteQuery/8cf560a3-f257-4d9d-900b-aaf46185fe8b \\u003e DN.FE.ExecuteQuery/7c0e685c-e2d3-436f-89bc-9d9fa3141b61)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[00:24:08.682]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:24:05.4122339Z);\\nlet _endTime = datetime(2026-08-25T00:24:05.4122344Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:24:08.6621602Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f\\\",\\r\\n            \\\"activityId\\\": \\\"ed3e04f4-6a49-4e81-9b00-d4558d9cddcc\\\",\\r\\n            \\\"subActivityId\\\": \\\"4cad9bba-6029-42b0-a77f-5c14bc4e8a84\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"58dfa59a-bdfb-430f-af23-104aaad94cc9\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f ARID=ed3e04f4-6a49-4e81-9b00-d4558d9cddcc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/58dfa59a-bdfb-430f-af23-104aaad94cc9 \\u003e DN.FE.ExecuteQuery/4cad9bba-6029-42b0-a77f-5c14bc4e8a84)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:24:08.778]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:24:05.4122339Z);\\nlet _endTime = datetime(2026-08-25T00:24:05.4122344Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:24:08.6621602Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f\\\",\\r\\n            \\\"activityId\\\": \\\"ed3e04f4-6a49-4e81-9b00-d4558d9cddcc\\\",\\r\\n            \\\"subActivityId\\\": \\\"4cad9bba-6029-42b0-a77f-5c14bc4e8a84\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"58dfa59a-bdfb-430f-af23-104aaad94cc9\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f ARID=ed3e04f4-6a49-4e81-9b00-d4558d9cddcc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/58dfa59a-bdfb-430f-af23-104aaad94cc9 \\u003e DN.FE.ExecuteQuery/4cad9bba-6029-42b0-a77f-5c14bc4e8a84)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.009s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00083c410\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[00:23:58.357]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:55.0351655Z);\\nlet _endTime = datetime(2026-08-25T00:23:55.0351661Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:23:58.3330999Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b\\\",\\r\\n            \\\"activityId\\\": \\\"442abb20-5490-45f5-ba70-3bca3bed0639\\\",\\r\\n            \\\"subActivityId\\\": \\\"b9132e8a-1c55-4f09-8982-f344852f0eee\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"99a04993-dace-423f-a879-2fbeee2a7b84\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b ARID=442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.Http.CallContext/442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.ExecuteQuery/99a04993-dace-423f-a879-2fbeee2a7b84 \\u003e DN.FE.ExecuteQuery/b9132e8a-1c55-4f09-8982-f344852f0eee)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:23:58.411]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:55.0351655Z);\\nlet _endTime = datetime(2026-08-25T00:23:55.0351661Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:23:58.3330999Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b\\\",\\r\\n            \\\"activityId\\\": \\\"442abb20-5490-45f5-ba70-3bca3bed0639\\\",\\r\\n            \\\"subActivityId\\\": \\\"b9132e8a-1c55-4f09-8982-f344852f0eee\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"99a04993-dace-423f-a879-2fbeee2a7b84\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;125f7c3f-8e3c-4adc-92d3-11f56e27a65b ARID=442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.Http.CallContext/442abb20-5490-45f5-ba70-3bca3bed0639 \\u003e GW.ExecuteQuery/99a04993-dace-423f-a879-2fbeee2a7b84 \\u003e DN.FE.ExecuteQuery/b9132e8a-1c55-4f09-8982-f344852f0eee)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[00:24:01.787]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:58.4821435Z);\\nlet _endTime = datetime(2026-08-25T00:23:58.4821442Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:24:01.7618170Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5\\\",\\r\\n            \\\"activityId\\\": \\\"d2adae1e-7609-4016-b166-5a8ff8be55d8\\\",\\r\\n            \\\"subActivityId\\\": \\\"7c0e685c-e2d3-436f-89bc-9d9fa3141b61\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"8cf560a3-f257-4d9d-900b-aaf46185fe8b\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5 ARID=d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.Http.CallContext/d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.ExecuteQuery/8cf560a3-f257-4d9d-900b-aaf46185fe8b \\u003e DN.FE.ExecuteQuery/7c0e685c-e2d3-436f-89bc-9d9fa3141b61)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:24:01.845]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:23:58.4821435Z);\\nlet _endTime = datetime(2026-08-25T00:23:58.4821442Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:24:01.7618170Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5\\\",\\r\\n            \\\"activityId\\\": \\\"d2adae1e-7609-4016-b166-5a8ff8be55d8\\\",\\r\\n            \\\"subActivityId\\\": \\\"7c0e685c-e2d3-436f-89bc-9d9fa3141b61\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"8cf560a3-f257-4d9d-900b-aaf46185fe8b\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;7425e14f-788f-421a-b19d-fdd4f5d966b5 ARID=d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.Http.CallContext/d2adae1e-7609-4016-b166-5a8ff8be55d8 \\u003e GW.ExecuteQuery/8cf560a3-f257-4d9d-900b-aaf46185fe8b \\u003e DN.FE.ExecuteQuery/7c0e685c-e2d3-436f-89bc-9d9fa3141b61)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[00:24:08.682]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:24:05.4122339Z);\\nlet _endTime = datetime(2026-08-25T00:24:05.4122344Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:24:08.6621602Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f\\\",\\r\\n            \\\"activityId\\\": \\\"ed3e04f4-6a49-4e81-9b00-d4558d9cddcc\\\",\\r\\n            \\\"subActivityId\\\": \\\"4cad9bba-6029-42b0-a77f-5c14bc4e8a84\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"58dfa59a-bdfb-430f-af23-104aaad94cc9\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f ARID=ed3e04f4-6a49-4e81-9b00-d4558d9cddcc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/58dfa59a-bdfb-430f-af23-104aaad94cc9 \\u003e DN.FE.ExecuteQuery/4cad9bba-6029-42b0-a77f-5c14bc4e8a84)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:24:08.778]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:24:05.4122339Z);\\nlet _endTime = datetime(2026-08-25T00:24:05.4122344Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-b5s7zllhqcpk'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:24:08.6621602Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f\\\",\\r\\n            \\\"activityId\\\": \\\"ed3e04f4-6a49-4e81-9b00-d4558d9cddcc\\\",\\r\\n            \\\"subActivityId\\\": \\\"4cad9bba-6029-42b0-a77f-5c14bc4e8a84\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"58dfa59a-bdfb-430f-af23-104aaad94cc9\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;6e8eb1dd-b3b2-4c08-bb8a-fabaa900a66f ARID=ed3e04f4-6a49-4e81-9b00-d4558d9cddcc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/7d4daa74-783c-457a-8256-bc6951fb2784 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/58dfa59a-bdfb-430f-af23-104aaad94cc9 \\u003e DN.FE.ExecuteQuery/4cad9bba-6029-42b0-a77f-5c14bc4e8a84)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.003s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000626a40\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[00:14:56.798]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:53.4068554Z);\\nlet _endTime = datetime(2026-08-25T00:14:53.4068559Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:14:56.7779164Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d\\\",\\r\\n            \\\"activityId\\\": \\\"697030e7-6169-48d4-afab-e55af5423283\\\",\\r\\n            \\\"subActivityId\\\": \\\"7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"e5629797-dc74-431c-89f2-26cc2ad935e1\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d ARID=697030e7-6169-48d4-afab-e55af5423283 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e5629797-dc74-431c-89f2-26cc2ad935e1 \\u003e DN.FE.ExecuteQuery/7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:14:56.846]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:53.4068554Z);\\nlet _endTime = datetime(2026-08-25T00:14:53.4068559Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:14:56.7779164Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d\\\",\\r\\n            \\\"activityId\\\": \\\"697030e7-6169-48d4-afab-e55af5423283\\\",\\r\\n            \\\"subActivityId\\\": \\\"7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"e5629797-dc74-431c-89f2-26cc2ad935e1\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d ARID=697030e7-6169-48d4-afab-e55af5423283 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e5629797-dc74-431c-89f2-26cc2ad935e1 \\u003e DN.FE.ExecuteQuery/7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[00:15:00.265]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:56.9537642Z);\\nlet _endTime = datetime(2026-08-25T00:14:56.9537648Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:15:00.2458657Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130\\\",\\r\\n            \\\"activityId\\\": \\\"60865211-a5af-4242-8bd8-14b7a1f199ba\\\",\\r\\n            \\\"subActivityId\\\": \\\"865c926e-620f-4107-9fd8-43ac9ee494b3\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"70a56a06-7a14-4b6c-b645-990ed51e93ae\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130 ARID=60865211-a5af-4242-8bd8-14b7a1f199ba \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/70a56a06-7a14-4b6c-b645-990ed51e93ae \\u003e DN.FE.ExecuteQuery/865c926e-620f-4107-9fd8-43ac9ee494b3)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:15:00.327]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:56.9537642Z);\\nlet _endTime = datetime(2026-08-25T00:14:56.9537648Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:15:00.2458657Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130\\\",\\r\\n            \\\"activityId\\\": \\\"60865211-a5af-4242-8bd8-14b7a1f199ba\\\",\\r\\n            \\\"subActivityId\\\": \\\"865c926e-620f-4107-9fd8-43ac9ee494b3\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"70a56a06-7a14-4b6c-b645-990ed51e93ae\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130 ARID=60865211-a5af-4242-8bd8-14b7a1f199ba \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/70a56a06-7a14-4b6c-b645-990ed51e93ae \\u003e DN.FE.ExecuteQuery/865c926e-620f-4107-9fd8-43ac9ee494b3)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[00:15:08.703]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:04.4342198Z);\\nlet _endTime = datetime(2026-08-25T00:15:04.4342235Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:15:08.6768100Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199\\\",\\r\\n            \\\"activityId\\\": \\\"a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19\\\",\\r\\n            \\\"subActivityId\\\": \\\"dda68eb2-c740-498f-b62e-c74a25b621f7\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"4d4ca23c-dcca-4b88-9136-446102508ccb\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199 ARID=a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.Http.CallContext/a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.ExecuteQuery/4d4ca23c-dcca-4b88-9136-446102508ccb \\u003e DN.FE.ExecuteQuery/dda68eb2-c740-498f-b62e-c74a25b621f7)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:15:08.705]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"context canceled\",\n      \"name\": \"cosmosResourceSnapshots\"\n    }�[0m\n    �[37m[00:15:09.096]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:04.4342198Z);\\nlet _endTime = datetime(2026-08-25T00:15:04.4342235Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:15:08.6768100Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199\\\",\\r\\n            \\\"activityId\\\": \\\"a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19\\\",\\r\\n            \\\"subActivityId\\\": \\\"dda68eb2-c740-498f-b62e-c74a25b621f7\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"4d4ca23c-dcca-4b88-9136-446102508ccb\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199 ARID=a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.Http.CallContext/a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.ExecuteQuery/4d4ca23c-dcca-4b88-9136-446102508ccb \\u003e DN.FE.ExecuteQuery/dda68eb2-c740-498f-b62e-c74a25b621f7)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:100]: Timed out after 600.003s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000626a40\u003e: \n    must-gather CLI test failures:\n      query-basic: command failed: exit status 1\n    output: �[37m[00:14:56.798]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:53.4068554Z);\\nlet _endTime = datetime(2026-08-25T00:14:53.4068559Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:14:56.7779164Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d\\\",\\r\\n            \\\"activityId\\\": \\\"697030e7-6169-48d4-afab-e55af5423283\\\",\\r\\n            \\\"subActivityId\\\": \\\"7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"e5629797-dc74-431c-89f2-26cc2ad935e1\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d ARID=697030e7-6169-48d4-afab-e55af5423283 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e5629797-dc74-431c-89f2-26cc2ad935e1 \\u003e DN.FE.ExecuteQuery/7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:14:56.846]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:53.4068554Z);\\nlet _endTime = datetime(2026-08-25T00:14:53.4068559Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:14:56.7779164Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d\\\",\\r\\n            \\\"activityId\\\": \\\"697030e7-6169-48d4-afab-e55af5423283\\\",\\r\\n            \\\"subActivityId\\\": \\\"7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"e5629797-dc74-431c-89f2-26cc2ad935e1\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;cb2874aa-23b9-4e90-82b8-8dbdc8ccb12d ARID=697030e7-6169-48d4-afab-e55af5423283 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/d94d70a1-2ca3-4462-b247-1c225640a5b0 \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/e5629797-dc74-431c-89f2-26cc2ad935e1 \\u003e DN.FE.ExecuteQuery/7025f3d5-21ad-4b66-b1ec-6016c4d8ba2f)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-skip-hcp-logs: command failed: exit status 1\n    output: �[37m[00:15:00.265]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:56.9537642Z);\\nlet _endTime = datetime(2026-08-25T00:14:56.9537648Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:15:00.2458657Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130\\\",\\r\\n            \\\"activityId\\\": \\\"60865211-a5af-4242-8bd8-14b7a1f199ba\\\",\\r\\n            \\\"subActivityId\\\": \\\"865c926e-620f-4107-9fd8-43ac9ee494b3\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"70a56a06-7a14-4b6c-b645-990ed51e93ae\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130 ARID=60865211-a5af-4242-8bd8-14b7a1f199ba \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/70a56a06-7a14-4b6c-b645-990ed51e93ae \\u003e DN.FE.ExecuteQuery/865c926e-620f-4107-9fd8-43ac9ee494b3)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:15:00.327]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:14:56.9537642Z);\\nlet _endTime = datetime(2026-08-25T00:14:56.9537648Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:15:00.2458657Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130\\\",\\r\\n            \\\"activityId\\\": \\\"60865211-a5af-4242-8bd8-14b7a1f199ba\\\",\\r\\n            \\\"subActivityId\\\": \\\"865c926e-620f-4107-9fd8-43ac9ee494b3\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"70a56a06-7a14-4b6c-b645-990ed51e93ae\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;d3045808-57d8-4c23-8f73-affb30f6a130 ARID=60865211-a5af-4242-8bd8-14b7a1f199ba \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e KD.Query.Client.ExecuteQueryAsKustoDataStream/230b95d6-3e5a-4f20-bb04-f8704206f9bc \\u003e P.Grpc.Service.ExecuteQueryAsKustoDataStream2/70a56a06-7a14-4b6c-b645-990ed51e93ae \\u003e DN.FE.ExecuteQuery/865c926e-620f-4107-9fd8-43ac9ee494b3)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n      query-collect-systemd-logs: command failed: exit status 1\n    output: �[37m[00:15:08.703]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:04.4342198Z);\\nlet _endTime = datetime(2026-08-25T00:15:04.4342235Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:15:08.6768100Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199\\\",\\r\\n            \\\"activityId\\\": \\\"a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19\\\",\\r\\n            \\\"subActivityId\\\": \\\"dda68eb2-c740-498f-b62e-c74a25b621f7\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"4d4ca23c-dcca-4b88-9136-446102508ccb\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199 ARID=a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.Http.CallContext/a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.ExecuteQuery/4d4ca23c-dcca-4b88-9136-446102508ccb \\u003e DN.FE.ExecuteQuery/dda68eb2-c740-498f-b62e-c74a25b621f7)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\",\n      \"name\": \"detailedInfraOrchestrationLogs\"\n    }�[0m\n    �[37m[00:15:08.705]�[0m �[91mERROR:�[0m �[97mQuery failed�[0m �[90m{\n      \"err\": \"context canceled\",\n      \"name\": \"cosmosResourceSnapshots\"\n    }�[0m\n    �[37m[00:15:09.096]�[0m �[91mERROR:�[0m �[97mcommand failed�[0m �[90m{\n      \"err\": \"failed to execute custom logs query: error during query execution: failed to execute query: failed to execute query: Op(OpQuery): Kind(KHTTPError): error from Kusto endpoint, With query: let _startTime = datetime(2026-08-24T00:15:04.4342198Z);\\nlet _endTime = datetime(2026-08-25T00:15:04.4342235Z);\\n// Extract cluster ID from CS logs using cid field\\nlet cluster_id = toscalar(\\n    database('ServiceLogs').table('clustersServiceLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where resource_id has 'kusto-logs-jk7hknvqffmd'\\n    | where isnotempty(log.cid)\\n    | distinct tostring(log.cid)\\n    | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n    database('ServiceLogs').table('containerLogs')\\n    | where timestamp between (_startTime .. _endTime)\\n    | where namespace_name has \\\"maestro\\\"\\n    | extend logStr = tostring(log)\\n    | where logStr has cluster_id\\n    | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n    | where isnotempty(resourceId)\\n    | distinct resourceId\\n);\\n// Infrastructure \\u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n    (\\n        // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n        // Includes: resource creation, updates, deletions, status events, gRPC communications\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"maestro\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id or logStr has_any (maestro_resource_ids)\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n        // Includes the CR reconciliation logs\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"hypershift\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    ),\\n    (\\n        // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n        // Includes: work agent operations, manifest application, status feedback\\n        database('ServiceLogs').table('containerLogs')\\n        | where timestamp between (_startTime .. _endTime)\\n        | where namespace_name has \\\"open-cluster-management-agent\\\"\\n        | extend logStr = tostring(log)\\n        | where logStr has cluster_id\\n        | project timestamp, container_name, msg = logStr, log\\n    )\\n| order by timestamp asc(400 Bad Request):\\n{\\r\\n    \\\"error\\\": {\\r\\n        \\\"code\\\": \\\"General_BadRequest\\\",\\r\\n        \\\"message\\\": \\\"Request is invalid and cannot be executed.\\\",\\r\\n        \\\"@type\\\": \\\"Kusto.Data.Exceptions.RelopSemanticException\\\",\\r\\n        \\\"@message\\\": \\\"Relop semantic error: SEM0026: The arguments array exceeded the allowed limit (allowed=10,000)\\\",\\r\\n        \\\"@failureCode\\\": 400,\\r\\n        \\\"@context\\\": {\\r\\n            \\\"timestamp\\\": \\\"2026-08-25T00:15:08.6768100Z\\\",\\r\\n            \\\"serviceAlias\\\": \\\"HCP-DEV-US-2\\\",\\r\\n            \\\"clientRequestId\\\": \\\"KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199\\\",\\r\\n            \\\"activityId\\\": \\\"a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19\\\",\\r\\n            \\\"subActivityId\\\": \\\"dda68eb2-c740-498f-b62e-c74a25b621f7\\\",\\r\\n            \\\"activityType\\\": \\\"DN.FE.ExecuteQuery\\\",\\r\\n            \\\"parentActivityId\\\": \\\"4d4ca23c-dcca-4b88-9136-446102508ccb\\\",\\r\\n            \\\"activityStack\\\": \\\"(Activity stack: CRID=KGC.execute;2c78ad4b-fe52-4282-9d01-7f8d1f03d199 ARID=a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.Http.CallContext/a7bce1a5-15e0-4dd5-9933-19a9ff7b6f19 \\u003e GW.ExecuteQuery/4d4ca23c-dcca-4b88-9136-446102508ccb \\u003e DN.FE.ExecuteQuery/dda68eb2-c740-498f-b62e-c74a25b621f7)\\\",\\r\\n            \\\"serviceFarm\\\": null\\r\\n        },\\r\\n        \\\"@permanent\\\": true\\r\\n    }\\r\\n}\"\n    }�[0m\n    \n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092034671041843200","occurred_at":"2026-08-24T23:41:48Z","signature_id":"93cd087eb0fc0ac2b0c79ba3bc6c6181f71cba9db1ac3eaefd4c1d8152fbb877","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092031971118026752","occurred_at":"2026-08-24T23:30:58Z","signature_id":"176c7f8e0e9a6fe4e5a12f9055659eede136ab94bf2bc621167d758c6dc1828c","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092030482601480192","occurred_at":"2026-08-24T23:25:09Z","signature_id":"a4d769910ea57a048ae6030e8851eaac93f508a9d62968944253d616d8fe88ed","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092025885698822144","occurred_at":"2026-08-24T23:06:53Z","signature_id":"9eb8d09c21becab11e4df2e2486fbadfcfe682a12484ff5957cc08b54e0cd902","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092020242174709760","occurred_at":"2026-08-24T22:44:28Z","signature_id":"a99ad7aa848b9f85d32a71fb35dfd49c3d8270fdc830d4478b6af42e4702a425","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092016236664721408","occurred_at":"2026-08-24T22:28:33Z","signature_id":"0d5221b00aeb7bfdca1da1581a9d7578e9d19b098899c13f87a46837f2394f22","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092014812023230464","occurred_at":"2026-08-24T22:22:53Z","signature_id":"c5fd89f1c41b331bfa0c3c5ff918b5ee94282a73960beb6c8680dfabd0d92aba","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2092003732647579648","occurred_at":"2026-08-24T21:38:47Z","signature_id":"c3920cca5c3b22e9ea0a8df63d7fd82dff370a2c49fd2a56d2b8612e6611e8cd","pr_number":6088,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091996785936437248","occurred_at":"2026-08-24T21:11:15Z","signature_id":"c79de5565353120aa973caf9da2e4e0043d28fd25ac56a7cd204827ac9e612e5","pr_number":6128,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091995273302642688","occurred_at":"2026-08-24T21:05:15Z","signature_id":"b19e1030a263db434fc5938127062d3add16c8d3a56319f83b1663785d30bba9","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"98f2c85bfff864b394f0a6540743e0876e4af34e9f265ec3b983e45e8ef64a57","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"fa1482e0c10fe89a03896ac5f14bc5ff11cbc2b5c024fd993535afa6403d322b","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091985911028912128","occurred_at":"2026-08-24T20:28:03Z","signature_id":"43910364f48f66523f1f9d08a417492faf101b0ab2cb444442ab1a553439c6b5","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091985362346840064","occurred_at":"2026-08-24T20:25:41Z","signature_id":"176cc4504c2f048683d4e9fd8346ddda26590ba46f5fcff44ce3756a85f5565e","pr_number":6662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","signature_id":"913731872ab076d2435270c0fedcc39b2d88a5ff36047161ca4eabe2b648b67d","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091978207036182528","occurred_at":"2026-08-24T19:57:24Z","signature_id":"f663b43ed0edc7e9f34c5b7d6058cf02871c4b2cbf7c46200eb2800bb395c1fc","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977503907254272","occurred_at":"2026-08-24T19:54:38Z","signature_id":"0b9a16666408f3f71c77e5a20e559de4e428d06efaede29056c5fdec679a0549","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977495384428544","occurred_at":"2026-08-24T19:54:36Z","signature_id":"07fcf787e9667bb548a1ae4e2aa892f82683b85aa913615ccc3c55f328e44f13","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977485536202752","occurred_at":"2026-08-24T19:54:34Z","signature_id":"edf4f7b0d5436e65624e47252a3c9819a947e53aeb06b94097e54f965d358ffa","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091965378614792192","occurred_at":"2026-08-24T19:06:19Z","signature_id":"69234d1190be6bae030c9f0be83f507930b8aba09ce9643964f39fe12af78ce2","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091963250454302720","occurred_at":"2026-08-24T18:58:00Z","signature_id":"1bd7b81142bf5c01b0963c412981b707f78cb8184adf320516339e4aa3a3ce9f","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091963205222928384","occurred_at":"2026-08-24T18:57:49Z","signature_id":"9bfada2b49b3fc5245efb1ec3e602fd604d8132a904f08045a044c170e893680","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091962153304068096","occurred_at":"2026-08-24T18:53:38Z","signature_id":"c742492599a708bf77717b7e4360828e91602b79b9e3f94c759bffe66f0abf90","pr_number":6648,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091960563839012864","occurred_at":"2026-08-24T18:47:19Z","signature_id":"ec35c9667d01d0fca7a963152324f60489cc7cd6ddc292c3798abbfb8aa51092","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091956227226472448","occurred_at":"2026-08-24T18:30:02Z","signature_id":"335052cc293f1dcfc9068c799f022fd529e07859d0cea778ae29ff1ed7adcc44","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091950955841982464","occurred_at":"2026-08-24T18:09:09Z","signature_id":"c0cebe2544e94f85956b2aa4e6cb37d39972915968a1824b87954f1d8f2b0278","pr_number":6661,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6529/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091948175697907712","occurred_at":"2026-08-24T17:58:06Z","signature_id":"2cfd9e8dd04e05dbf25d7770f4e1349a8d7b20bfb77b1b160b57373b94af3940","pr_number":6529,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091942092610736128","occurred_at":"2026-08-24T17:33:55Z","signature_id":"41ffa53c8fc1675a8e8f11ea4fff86270a85dca8d99ed458236b7319ea0aa4d4","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091939168878858240","occurred_at":"2026-08-24T17:22:18Z","signature_id":"dec2f42fbc616357b928df0a2e19436696b2e782470cc0447de0e33d58d63917","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091932257060130816","occurred_at":"2026-08-24T16:54:39Z","signature_id":"cff96711d2e8bb5b9f48911a18c760f7b843abef0ee0b1364775c2b0b5f58a49","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091932359686361088","occurred_at":"2026-08-24T16:54:34Z","signature_id":"5fa5e4ca1b4d0f869a74c660939188cb0fcaacbc16b74da34e727232c29abbff","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"6ff77bfa149b82860b0429a7dfe5ab2b71a58982ddc24dc6fbf5996e7d1dca8d","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"1ae3c1ffd864572007c734910a8d27ab7b2bea356002ed663c85c3e98b525248","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"9865c7135fd03ceb69a76994157b6614c9b64d8eebb69b3b00180111a55f75a4","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091926769790619648","occurred_at":"2026-08-24T16:21:04Z","signature_id":"40f2371fc4b699bfbeed8a52087620105728edb5da70261d5e9bd7df99b13e6a","pr_number":6639,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":34,"runs_affected":34,"failed_runs":34,"run_impact_percent":5.792163543441227,"anchor_occurrences":34,"after_last_push_count":7,"trend_counts":[0,0,0,0,6,0,0,28],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":34,"prior_last_seen_at":"2026-08-21T03:51:03Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":8},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":5},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":21}],"full_error_samples":["time=2026-08-24T13:26:56.540Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T13:26:58.762Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T13:27:00.485Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=abeb20b90f6fb4be4d4a69c08ed3bbe8b5be5f3247506ae425e325b85b345969 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8084480-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fabeb20b90f6fb4be4d4a69c08ed3bbe8b5be5f3247506ae425e325b85b345969\ntime=2026-08-24T13:35:32.482Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.Resources/deployments/abeb20b90f6fb4be4d4a69c08ed3bbe8b5be5f3247506ae425e325b85b345969/operationStatuses/08584140288659502671\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.Resources/deployments/cluster-kyv757fphd2fm\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j8084480-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n                    \\\\\\\"details\\\\\\\": [\\\\r\\\\n                      {\\\\r\\\\n                        \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n                        \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-24652102-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8084480-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-24652102-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n                        \\\\\\\"details\\\\\\\": [\\\\r\\\\n                          {\\\\r\\\\n                            \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n                            \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n                            \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n                          }\\\\r\\\\n                        ]\\\\r\\\\n                      }\\\\r\\\\n                    ]\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-08-24T13:31:41.774Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T13:31:42.863Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-08-24T13:31:44.020Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=cf8b4dcb00b286eb41bc03071739c50d179926b7125e9927b43e768ae20b10a5 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5557248-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fcf8b4dcb00b286eb41bc03071739c50d179926b7125e9927b43e768ae20b10a5\ntime=2026-08-24T13:39:15.842Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.Resources/deployments/cf8b4dcb00b286eb41bc03071739c50d179926b7125e9927b43e768ae20b10a5/operationStatuses/08584140285820210806\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.Resources/deployments/cluster-ce5kkcjcnxbu2\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.Resources/deployments/infra-agent-pools\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5557248-mgmt-2/agentPools/infra1\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n                    \\\\\\\"details\\\\\\\": [\\\\r\\\\n                      {\\\\r\\\\n                        \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n                        \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-20013346-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5557248-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-20013346-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n                        \\\\\\\"details\\\\\\\": [\\\\r\\\\n                          {\\\\r\\\\n                            \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n                            \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n                            \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n                          }\\\\r\\\\n                        ]\\\\r\\\\n                      }\\\\r\\\\n                    ]\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-08-24T13:26:28.921Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-24T13:26:30.957Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-24T13:26:32.602Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=9508f110ddf9b49e79a1c674ffa6223d05182dbd61375e2ba689c04c8113e848 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2132480-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9508f110ddf9b49e79a1c674ffa6223d05182dbd61375e2ba689c04c8113e848\ntime=2026-08-24T13:34:04.585Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.Resources/deployments/9508f110ddf9b49e79a1c674ffa6223d05182dbd61375e2ba689c04c8113e848/operationStatuses/08584140288938416315\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.Resources/deployments/cluster-d54rkmqgu5xlq\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j2132480-svc/agentPools/u64d8dsv61\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n                    \\\\\\\"details\\\\\\\": [\\\\r\\\\n                      {\\\\r\\\\n                        \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n                        \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-42034125-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2132480-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-u64d8dsv61-42034125-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n                        \\\\\\\"details\\\\\\\": [\\\\r\\\\n                          {\\\\r\\\\n                            \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n                            \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\r\\\\n                            \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n                          }\\\\r\\\\n                        ]\\\\r\\\\n                      }\\\\r\\\\n                    ]\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091873075288084480","occurred_at":"2026-08-24T12:59:40Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091872721045557248","occurred_at":"2026-08-24T12:58:16Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091872693082132480","occurred_at":"2026-08-24T12:58:09Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091872666397970432","occurred_at":"2026-08-24T12:58:03Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091868107806085120","occurred_at":"2026-08-24T12:39:55Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6585,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091867251824136192","occurred_at":"2026-08-24T12:36:32Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6585,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091856057667162112","occurred_at":"2026-08-24T11:52:03Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6585,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6424/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091852685220253696","occurred_at":"2026-08-24T11:34:18Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6424,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091844440330080256","occurred_at":"2026-08-24T11:05:53Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091844377495212032","occurred_at":"2026-08-24T11:05:38Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091844284889174016","occurred_at":"2026-08-24T11:05:16Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091840235057975296","occurred_at":"2026-08-24T10:49:11Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091827867787726848","occurred_at":"2026-08-24T10:00:02Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091827277728845824","occurred_at":"2026-08-24T09:57:41Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6429,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091823405572558848","occurred_at":"2026-08-24T09:42:18Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091816375096971264","occurred_at":"2026-08-24T09:14:22Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6429,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6517/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091814034880860160","occurred_at":"2026-08-24T09:05:04Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6517,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091812979992432640","occurred_at":"2026-08-24T09:00:53Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091803125458407424","occurred_at":"2026-08-24T08:21:34Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091801419551084544","occurred_at":"2026-08-24T08:14:51Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091795137024757760","occurred_at":"2026-08-24T07:49:58Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091792609700745216","occurred_at":"2026-08-24T07:39:56Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6278,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6656/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091776267920609280","occurred_at":"2026-08-24T06:35:00Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6656,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091774664165560320","occurred_at":"2026-08-24T06:28:37Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091750724990930944","occurred_at":"2026-08-24T04:53:30Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6503,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091747201922371584","occurred_at":"2026-08-24T04:39:30Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091734160543584256","occurred_at":"2026-08-24T03:47:41Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091733541242015744","occurred_at":"2026-08-24T03:45:13Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6646,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090647847190925312","occurred_at":"2026-08-21T03:51:03Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6619,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090611472622161920","occurred_at":"2026-08-21T01:26:31Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090609877989724160","occurred_at":"2026-08-21T01:20:11Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090609408819073024","occurred_at":"2026-08-21T01:18:19Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090605238405828608","occurred_at":"2026-08-21T01:01:45Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090596628044976128","occurred_at":"2026-08-21T00:27:32Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6278,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":31,"runs_affected":31,"failed_runs":31,"run_impact_percent":5.281090289608177,"anchor_occurrences":31,"after_last_push_count":29,"trend_counts":[9,10,5,6,5,3,0,2],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":124,"prior_last_seen_at":"2026-08-22T08:48:46Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":31}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T18:01:25Z\n  Ended:       2026-08-24T18:34:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T18:01:25Z\n  Ended:       2026-08-24T18:34:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T17:55:40Z\n  Ended:       2026-08-24T18:14:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0629888-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.209:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0629888/providers/microsoft.monitor/accounts/hcps-j0629888\", namespace=\"ocm-arohcpci01-2sd354mcvucng8mm6igmcq4k43caf9jb-s0d4l4d8t6j0p0t\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-tktc6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sd354mcvucng8mm6igmcq4k43caf9jb-s0d4l4d8t6j0p0t/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T17:55:40Z\n  Ended:       2026-08-24T18:15:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0629888-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.209:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0629888/providers/microsoft.monitor/accounts/hcps-j0629888\", namespace=\"ocm-arohcpci01-2sd354mcvucng8mm6igmcq4k43caf9jb-s0d4l4d8t6j0p0t\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-tktc6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sd354mcvucng8mm6igmcq4k43caf9jb-s0d4l4d8t6j0p0t/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-22T09:51:47Z\n  Ended:       2026-08-22T10:13:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3442688-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.129:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3442688/providers/microsoft.monitor/accounts/hcps-j3442688\", namespace=\"ocm-arohcpci01-2sbhtcj4djmllishlrj1ubeicnu0g5pf-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-hp77b\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sbhtcj4djmllishlrj1ubeicnu0g5pf-sre-hcp-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-22T09:52:47Z\n  Ended:       2026-08-22T10:09:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3442688-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.129:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3442688/providers/microsoft.monitor/accounts/hcps-j3442688\", namespace=\"ocm-arohcpci01-2sbhtcj4djmllishlrj1ubeicnu0g5pf-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-hp77b\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2sbhtcj4djmllishlrj1ubeicnu0g5pf-sre-hcp-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"97ee0c31f908f1dbb20c9b6757ed42d4855cc07c6e4c69aed451babbcc2b480c","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"99e8b0c7fce35f618653d3b9144ebffd6644e1f77bf1ee7a7e41e85ca3b0c8eb","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091085161763442688","occurred_at":"2026-08-22T08:48:46Z","signature_id":"23d14823c199d5901c0285f35564ec7461e2aa45ffdc1a27b21665c828219f0b","pr_number":6640,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"22183042c9ed30b84dbf6d39061ab4bc9df56e90ac5267a78267ac4493dba186","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090960143322714112","occurred_at":"2026-08-22T00:32:00Z","signature_id":"7beacbaf5f0e4bfbba1a2f2ad49cee0d638516eb819a023195eabd3599d5ec0e","pr_number":6639,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"46d4ece5f5d566a68e9c280231b88623d89a58a8337d07f4f4a7df19acc9fd13","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090912033770835968","occurred_at":"2026-08-21T21:20:50Z","signature_id":"99fe7c2c7c806ddd200c62659c7ae464504909d81a4495a62d531c5eefa98193","pr_number":6651,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090807183972241408","occurred_at":"2026-08-21T14:24:12Z","signature_id":"80ee9143b08193bf0a21a4ec6c3243441efc3dbf08661b31e12362f00673d7e1","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090669519650426880","occurred_at":"2026-08-21T05:17:10Z","signature_id":"2de1f116a35eddb03dc2ba7d4574fd71bc2baa693fc063c5ce35ef13b561da68","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"04de974286e7d1201173912d76047280a42dd798689b1612cc47a67ded0d727b","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"6ae1089324989f77aeb5a95ad4a7163481d5b47367c03569c71e81b3ac2beea6","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"4ef8d0aefb0cc453e9e9e28247d6a5123fcebf0ff7b94c102214341e80194200","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090497381098328064","occurred_at":"2026-08-20T17:53:09Z","signature_id":"6d7eaaa13690b75238e05b7d352b6a42cd8b1a10a0bdd33fded04570e22f0078","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090465920358027264","occurred_at":"2026-08-20T15:48:08Z","signature_id":"fc4a65ca072851cf04eb2c3585332a115cfca96e64ca1e278a1ec250c0a7be60","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090405439429677056","occurred_at":"2026-08-20T11:47:38Z","signature_id":"b4e4fcfee50aa357f28aa6f512f29bb34e2833c6f3129d1415d2343a83b72874","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"549491e61691d2966c0af2b0191ce2097e27428f0a5ae908e536be79c7c164e7","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090191599463043072","occurred_at":"2026-08-19T21:38:05Z","signature_id":"cf00ab67daf9fe871c7e62c5a966fc1052eca72c90f665ad5a93628612da6e0a","pr_number":6608,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"361977dd6db3385463de7b49ec92e3b08b56667c2651facab94103ddbe36fe62","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090133175878553600","occurred_at":"2026-08-19T17:45:52Z","signature_id":"2b8c33cd66ab1717f619ed8958087c1440b2c477957c77056db36eba09ec220b","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090085264507539456","occurred_at":"2026-08-19T14:35:33Z","signature_id":"ffa6c82dc196170eafb22839b837c7e113ebd218f7e6741760d68be504b62d8d","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"4ff4aa5d104b7c3ca3e7f62fe40712f5a28a0dd3283492fb02c08dfec1c5ece5","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089829163719462912","occurred_at":"2026-08-18T21:37:54Z","signature_id":"2611722e20bf6b04b4ec96b6e94d280ca603e25cb0684c613e83b18d37aa8be7","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089759198756737024","occurred_at":"2026-08-18T16:59:53Z","signature_id":"e018dac98ac37d019854cff2dfa8a0af84126ecd12c21cc82547c4ea28b9b6a6","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089754858566455296","occurred_at":"2026-08-18T16:42:38Z","signature_id":"78a386fc73699ca7e911cdff440310696128cca04a45acc40be9a5ecf77716bc","pr_number":6553,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089753382901256192","occurred_at":"2026-08-18T16:36:46Z","signature_id":"d718eb78b7e9d9933cac0a1da3a80ea863953b9fb3cb3e720acbf5a2e538a8cf","pr_number":5900,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"ee74e2b4c4fa00a680ed8a55a00f5044c6fda849a4c0f29b640ae0bac4792f23","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089723779814002688","occurred_at":"2026-08-18T14:39:04Z","signature_id":"885dbcf836ffcf95b26fe4eb7750e521eb6934b59566dc5259806714fb8286e9","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089713360550498304","occurred_at":"2026-08-18T13:57:44Z","signature_id":"eafdc0673228e648346cde70fda85abaf4e4f34c3efc3d3bba757c76b1b53a74","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708056525934592","occurred_at":"2026-08-18T13:36:10Z","signature_id":"6c27eff3ad99a3534fd3aec1b6e86f5bbaf569fc81f297f020eb3883f46ff4f7","pr_number":6585,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6426/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089706907987087360","occurred_at":"2026-08-18T13:32:06Z","signature_id":"7a10d7261b620e2eb80897e88c86ffa09966d8a7a84589266a57ddd86bcb6448","pr_number":6426,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"b2d69a43faa68a4dd1d84e3e22ab7a98f47f4471536b04ee1f205fe32a332854","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"519fe0fe7478bebfb7ef939c84f76fa0281886d85cb376148838a1878147b618","failure_pattern":"alert [svc] KubeNodeUnreachable fired","search_query":"alert [svc] KubeNodeUnreachable fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeUnreachable does not fire","occurrences":28,"runs_affected":28,"failed_runs":28,"run_impact_percent":4.770017035775128,"anchor_occurrences":28,"after_last_push_count":53,"trend_counts":[12,7,4,7,9,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":172,"prior_last_seen_at":"2026-08-21T21:25:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeNodeUnreachable does not fire","occurrences":28}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T16:12:59Z\n  Ended:       2026-08-24T16:22:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"prometheus\", node=\"aks-userswft2-20578738-vmss000002\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: aks-userswft2-20578738-vmss000002 is unreachable and some workloads may be rescheduled.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T16:12:59Z\n  Ended:       2026-08-24T16:23:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"prometheus\", node=\"aks-userswft2-20578738-vmss000002\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-wgnw8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: aks-userswft2-20578738-vmss000002 is unreachable and some workloads may be rescheduled.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"prometheus\", node=\"aks-userswft1-18345013-vmss000002\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: aks-userswft1-18345013-vmss000002 is unreachable and some workloads may be rescheduled.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:49:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j0088320-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0088320/providers/microsoft.monitor/accounts/services-j0088320\", namespace=\"prometheus\", node=\"aks-userswft1-18345013-vmss000002\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-vxkdd\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: aks-userswft1-18345013-vmss000002 is unreachable and some workloads may be rescheduled.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"prometheus\", node=\"aks-userswft1-16379006-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: aks-userswft1-16379006-vmss000000 is unreachable and some workloads may be rescheduled.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T22:48:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j0835968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0835968/providers/microsoft.monitor/accounts/services-j0835968\", namespace=\"prometheus\", node=\"aks-userswft1-16379006-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-cxxjw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: aks-userswft1-16379006-vmss000000 is unreachable and some workloads may be rescheduled."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"8ea4612bb5240c0f449b27c94dc170f61dc652ef24bc74880e6489a1d15ea2be","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090913249020088320","occurred_at":"2026-08-21T21:25:40Z","signature_id":"962e0ef010bf48301058ca962542e7ac5e9edc6b0a882c5a67b669772b738665","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090912033770835968","occurred_at":"2026-08-21T21:20:50Z","signature_id":"2521e3c087abfbb2e0d499e90735681715a2dee5f344667b798d78dfae706547","pr_number":6651,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894675127832576","occurred_at":"2026-08-21T20:11:52Z","signature_id":"9d08db6e6be5de57ec10306b0d33b8e15ea40e6a5278009d21b49695e9d5a681","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090849031038177280","occurred_at":"2026-08-21T17:10:29Z","signature_id":"5ea76d1d2bb1fdd8bb56e9971199983dd7068f7bf46c775ec6d1ed78ae8978a2","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"07eaf5ac7c17bbce04470e8a1727fe0e2aa5c26739adb10fb318a6e08fe07b87","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090676319984553984","occurred_at":"2026-08-21T05:44:12Z","signature_id":"fd2db70b409a9f3665fa7cca11faadb591d9872b90277565d9b8d6734403499b","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657643948412928","occurred_at":"2026-08-21T04:29:59Z","signature_id":"136d54f78fc7a5cdc61527db90718c6d08f8a61c1f81807bda338e6b6d2c846f","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"88c75debdf1c34c29f7554793a0f52ff02bca445bb3ca24bd0878181de5f754e","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090612430223708160","occurred_at":"2026-08-21T01:30:11Z","signature_id":"209558f071f2f3b6cc35a68d8a5e17f16b7be58a627ef1f7287a910f7660c1c8","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090587456947621888","occurred_at":"2026-08-20T23:51:05Z","signature_id":"0fed287722e4ffab1c9718ca7b358dc431bfb749907bd990368d17d5f11fc4ae","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"fad068e6b6e4e1c995deb946f615a41fe51a51a10f0f5201c087eb71241c9966","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"da4cdd5d4991684fed1694b77cabfbc48bafe4370fec990007869d97caba68ca","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090443849187987456","occurred_at":"2026-08-20T14:20:26Z","signature_id":"dd837c8f3fd95bbca2b7933a9995248d0eb0017f33e5c8e584e0c53de04cb4b4","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6589/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090373215313465344","occurred_at":"2026-08-20T09:39:46Z","signature_id":"8299834d2efef2afff12de3e63ac69e61454ca342d920e15af7739595fdd39f9","pr_number":6589,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"b7da6f615fe5b11f0175e23954d32cde52017be4d8f9772defa4988af94e4830","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"8d04a3b31b14f6560bf5f8ba4c6c9d73c523676a77466f47e8eca833c27b2ea5","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"d4590bf5ab58e49323e9d4bbf719388cdb49a408dd121dee64f60e49573bdf2b","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"15f836f9d22cb527e0cd53b6fa63431a2ec2adc9699b9f5f0cf7b5b28f0e1858","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090035242617278464","occurred_at":"2026-08-19T11:16:37Z","signature_id":"85551ce531dc29c4032a21ea6ee815ac220ea9956f58c43585d642bfb6dea089","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6602/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090018285352390656","occurred_at":"2026-08-19T10:09:24Z","signature_id":"c262a21745c515132de6567b1de37663961ce91df589b275b9aeae9e6384874f","pr_number":6602,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089829163719462912","occurred_at":"2026-08-18T21:37:54Z","signature_id":"f897f0e473fb08907f312699bfca71343fbcd5309e397b5aaaba6fc59df6fe06","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089809279165927424","occurred_at":"2026-08-18T20:18:53Z","signature_id":"d601db77aaf93f7ecd94f9f311971c7337e5406b353e3760be6b59000f28b3c9","pr_number":6562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089759198756737024","occurred_at":"2026-08-18T16:59:53Z","signature_id":"6babc2af4fa5c9a2df7d1af571d762d97219bcffdd0ccd781fa3d335fb7d4acf","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089757851600818176","occurred_at":"2026-08-18T16:54:32Z","signature_id":"1a00055cde59c4642fb0c494ec8d98e6a03a84d8c8e3d1f57f6741ed2a7ffef4","pr_number":5880,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089723779814002688","occurred_at":"2026-08-18T14:39:04Z","signature_id":"5ebd64af0d4dc568c229dcfab7bf7f5fec55f46304592661f19577f285c704f5","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"e53b489507eca9c431c8baaaf69831b56ba39d78bd12e9a077823e5e9942fa41","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089651610870353920","occurred_at":"2026-08-18T09:52:22Z","signature_id":"2fc85af1031e84749b572ccb011e75151838636b2abcaae53a551e65d140dd99","pr_number":6562,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"a4b5d2ceab7788d5559bbf63902d6a202745bcaeee7459ce85981f7449607117","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":17,"runs_affected":8,"failed_runs":8,"run_impact_percent":1.362862010221465,"anchor_occurrences":17,"after_last_push_count":13,"also_in":["INT","PROD","STG"],"trend_counts":[2,11,2,0,0,4,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":22,"prior_last_seen_at":"2026-08-22T00:10:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009a11a0\u003e: \n    failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-s68dh8vhmfjq\" to finish creating: GET https://rp.j1593344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/41fcc53a-afde-489d-8eb8-76949cdd9b00\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/41fcc53a-afde-489d-8eb8-76949cdd9b00\",\n      \"name\": \"41fcc53a-afde-489d-8eb8-76949cdd9b00\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:18:54.359713618Z\",\n      \"endTime\": \"2026-08-22T01:38:02.522784991Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009a11a0\u003e: \n    failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-s68dh8vhmfjq\" to finish creating: GET https://rp.j1593344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/41fcc53a-afde-489d-8eb8-76949cdd9b00\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/41fcc53a-afde-489d-8eb8-76949cdd9b00\",\n      \"name\": \"41fcc53a-afde-489d-8eb8-76949cdd9b00\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:18:54.359713618Z\",\n      \"endTime\": \"2026-08-22T01:38:02.522784991Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-kbpdrprmz7mq/cluster-candidate-4-21-b4jg8c should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001c1c820\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-b4jg8c: failed waiting for cluster=\"cluster-candidate-4-21-b4jg8c\" in resourcegroup=\"rg-candidate-4-21-kbpdrprmz7mq\" to finish creating: GET https://rp.j1593344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5710b43e-bf93-4eff-a856-1083a1a7d12c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5710b43e-bf93-4eff-a856-1083a1a7d12c\",\n      \"name\": \"5710b43e-bf93-4eff-a856-1083a1a7d12c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:17:59.540344003Z\",\n      \"endTime\": \"2026-08-22T01:37:02.483270247Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-kbpdrprmz7mq/cluster-candidate-4-21-b4jg8c should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001c1c820\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-b4jg8c: failed waiting for cluster=\"cluster-candidate-4-21-b4jg8c\" in resourcegroup=\"rg-candidate-4-21-kbpdrprmz7mq\" to finish creating: GET https://rp.j1593344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5710b43e-bf93-4eff-a856-1083a1a7d12c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5710b43e-bf93-4eff-a856-1083a1a7d12c\",\n      \"name\": \"5710b43e-bf93-4eff-a856-1083a1a7d12c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:17:59.540344003Z\",\n      \"endTime\": \"2026-08-22T01:37:02.483270247Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f82620\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-h2svf5g2kxfd\" to finish creating: GET https://rp.j1593344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/50d4343e-33b2-4cbc-adf4-8b66762f4702\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/50d4343e-33b2-4cbc-adf4-8b66762f4702\",\n      \"name\": \"50d4343e-33b2-4cbc-adf4-8b66762f4702\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:19:57.95592072Z\",\n      \"endTime\": \"2026-08-22T01:39:02.502727987Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f82620\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-h2svf5g2kxfd\" to finish creating: GET https://rp.j1593344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/50d4343e-33b2-4cbc-adf4-8b66762f4702\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/50d4343e-33b2-4cbc-adf4-8b66762f4702\",\n      \"name\": \"50d4343e-33b2-4cbc-adf4-8b66762f4702\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:19:57.95592072Z\",\n      \"endTime\": \"2026-08-22T01:39:02.502727987Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090954745601593344","occurred_at":"2026-08-22T00:10:34Z","signature_id":"9fdd133af6242db076a4b863aa1d464f03e0be3d2425306bd6ebd2a2737c472d","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090954745601593344","occurred_at":"2026-08-22T00:10:34Z","signature_id":"ca8616230c8094e8215035c77732ff5477c5550d89406ec868bafc87db992ea3","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090954745601593344","occurred_at":"2026-08-22T00:10:34Z","signature_id":"d4f90a79a8a4ef80c12861acf6f718ca11e3feb268ed3a504ce6369876d11459","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090954745601593344","occurred_at":"2026-08-22T00:10:34Z","signature_id":"fc5eacd8f09f9d4725654010946fc92d203916fc49bf7d302d0ea51293543de8","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"71f14ae85372860033787bd468b4f0331d04c4854ae7cf27ac26f9ab2d4916bd","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"8b23ec894c227e8f12470a818eb300ee9ff167afc7d296ad5b545a1ec14f8a5e","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089786820278620160","occurred_at":"2026-08-18T18:48:55Z","signature_id":"617d11f0646dfcc261bd124d16c59f39e0f94486ccb97ec1b7d7d6bcccd2eae3","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089786820278620160","occurred_at":"2026-08-18T18:48:55Z","signature_id":"66d60e0964973b0fc0273fa0b826f544095354d1d18b4d2b73feeca8668f1921","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089740517767122944","occurred_at":"2026-08-18T15:45:39Z","signature_id":"1ce5c2601b4deb6da45edc73850c6f6073074ef8e4221d51193a25e5a858cf4e","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089740517767122944","occurred_at":"2026-08-18T15:45:39Z","signature_id":"63a5223436112d9a7c2ee04d630303801b6e6cb9b4c44636580af56e050f6179","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089740517767122944","occurred_at":"2026-08-18T15:45:39Z","signature_id":"e006cd35963912f319cbbdd159aa6a038cf404ad03454f491fecb3a41b559430","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"9a367074a3edd35554d25f96b05bc1de80a996afd9163d415810f712009df412","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"bf75033af047cab658c57ab96f4970f22557c6ff27976da003094207ae361ff7","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6564/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725877364461568","occurred_at":"2026-08-18T14:47:25Z","signature_id":"72703d7f2ad4c68d705f0a32aebd34a57c46d6e2207e7d4c694603041ae3cef0","pr_number":6564,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6564/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725877364461568","occurred_at":"2026-08-18T14:47:25Z","signature_id":"c31c861795d0a7106de988c48e228efc91827fe323d2ae01d4bce6df3828e503","pr_number":6564,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6586/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708355189739520","occurred_at":"2026-08-18T13:36:31Z","signature_id":"c99e87d9b77e33073832274b7b96ac9134240e0ae7c458660e4baad68f1d81bc","pr_number":6586,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089665224855523328","occurred_at":"2026-08-18T10:46:28Z","signature_id":"e6708d23e7e4f3700cf745774a7d1373fcc35ba88471be96741b14b55624b548","pr_number":6573,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"751c4f1c2065a85f77bda1ad36adb7417adf3ba06116dc7086a390665ad8d273","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":15,"runs_affected":15,"failed_runs":15,"run_impact_percent":2.5553662691652472,"anchor_occurrences":15,"after_last_push_count":15,"also_in":["INT","PROD","STG"],"trend_counts":[1,2,1,7,2,1,0,2],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":46,"prior_last_seen_at":"2026-08-22T00:13:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:166]: failed to create node pool npupgrade-4-20 with version 4.20.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00031f900\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-tczrqg\" in resourcegroup=\"rg-np-version-upgrade-tczrqg-fhlnbg67qjlz\" to finish creating: GET https://rp.j7254272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2d7b313-a7f0-40de-93b0-918f61dd2377\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2d7b313-a7f0-40de-93b0-918f61dd2377\",\n      \"name\": \"c2d7b313-a7f0-40de-93b0-918f61dd2377\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T20:58:35.24134299Z\",\n      \"endTime\": \"2026-08-24T21:17:39.53487523Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:166]: failed to create node pool npupgrade-4-20 with version 4.20.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00031f900\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-tczrqg\" in resourcegroup=\"rg-np-version-upgrade-tczrqg-fhlnbg67qjlz\" to finish creating: GET https://rp.j7254272.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2d7b313-a7f0-40de-93b0-918f61dd2377\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2d7b313-a7f0-40de-93b0-918f61dd2377\",\n      \"name\": \"c2d7b313-a7f0-40de-93b0-918f61dd2377\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T20:58:35.24134299Z\",\n      \"endTime\": \"2026-08-24T21:17:39.53487523Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:599]: failed to create nodepool npdg-4-21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e96960\u003e: \n    failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-q2tthc\" in resourcegroup=\"rg-np-downgrade-q2tthc-z7rbl7r2jwws\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47831462-3285-4a15-8722-8601b5708cf5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47831462-3285-4a15-8722-8601b5708cf5\",\n      \"name\": \"47831462-3285-4a15-8722-8601b5708cf5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:29:26.286804415Z\",\n      \"endTime\": \"2026-08-24T15:48:34.685207308Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:599]: failed to create nodepool npdg-4-21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e96960\u003e: \n    failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-q2tthc\" in resourcegroup=\"rg-np-downgrade-q2tthc-z7rbl7r2jwws\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47831462-3285-4a15-8722-8601b5708cf5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47831462-3285-4a15-8722-8601b5708cf5\",\n      \"name\": \"47831462-3285-4a15-8722-8601b5708cf5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:29:26.286804415Z\",\n      \"endTime\": \"2026-08-24T15:48:34.685207308Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:192]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0020dd2d8\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-m6p9wwz6cdqp\" to finish creating: GET https://rp.j4608896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bace4bf7-d284-4ee6-a7e4-bdbf462cf64f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bace4bf7-d284-4ee6-a7e4-bdbf462cf64f\",\n      \"name\": \"bace4bf7-d284-4ee6-a7e4-bdbf462cf64f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:31:06.356215706Z\",\n      \"endTime\": \"2026-08-22T01:50:12.095890348Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:192]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0020dd2d8\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-m6p9wwz6cdqp\" to finish creating: GET https://rp.j4608896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bace4bf7-d284-4ee6-a7e4-bdbf462cf64f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bace4bf7-d284-4ee6-a7e4-bdbf462cf64f\",\n      \"name\": \"bace4bf7-d284-4ee6-a7e4-bdbf462cf64f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:31:06.356215706Z\",\n      \"endTime\": \"2026-08-22T01:50:12.095890348Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977503907254272","occurred_at":"2026-08-24T19:54:38Z","signature_id":"eb508bccb119e029161a871dee87adcf17a97ac2d0acf65b89eac6aede0f896a","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"29b0688020a02097d8409be35d56bc8e657d156f249e7e0111a076d22256e797","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6593/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090955578854608896","occurred_at":"2026-08-22T00:13:52Z","signature_id":"f723c1595727953323cb9fbfda183c1dc163da8a20f83d112c16bfb48a359e65","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090803404556210176","occurred_at":"2026-08-21T14:09:11Z","signature_id":"5d29df6edd24bf24825c111a643726aa35b3e47b76bb21cfd88adb9f8c198cf4","pr_number":6635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090612430223708160","occurred_at":"2026-08-21T01:30:11Z","signature_id":"b92b8c9c37c7e37c897bea63a81f66153460a282d26942ce204cb4ed880d97f4","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090576352783634432","occurred_at":"2026-08-20T23:06:58Z","signature_id":"c1ea50b77f344e91f10a7709d63b68ac571c1fae52ae6c557bcbde6d0e3e6973","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"25c1614b8b68688875cf44e728f0eac99596a1b24838f886e170d041538bea9f","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090501897923334144","occurred_at":"2026-08-20T18:10:59Z","signature_id":"27c839aa327e50470cf56ca5936906afe6a900b7f124103d58137e60ab9f15a0","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090497381098328064","occurred_at":"2026-08-20T17:53:09Z","signature_id":"b0e35cbedbf30d1fba69356b51c668e445c0f866abb5ade28eeb3cbf4e7b6dac","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090484744067223552","occurred_at":"2026-08-20T17:02:48Z","signature_id":"0f90aceec820f2d550b3126a4ae2a3020109e8539c5f0cdecebafb3137413a11","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090483907651702784","occurred_at":"2026-08-20T16:59:32Z","signature_id":"859b652c2c687397467c692aff5a0fb14dd7e1b66f94076a5f0315ad529f937a","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6589/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090373215313465344","occurred_at":"2026-08-20T09:39:46Z","signature_id":"501dd06fef250a9b340590187f616f2aef38d9c4cc0b4b2d3fcf24ce1e2979fb","pr_number":6589,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089981796769861632","occurred_at":"2026-08-19T07:44:24Z","signature_id":"2a8c3f17b5d44463f41d98cc3eb1095019ba876a0c688336c2ff2416d9d79832","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089820335682621440","occurred_at":"2026-08-18T21:02:49Z","signature_id":"99d79d9bbd815eb5061871cab56284afb64f460af8f4dd07f3f38206fe27d279","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089679544620421120","occurred_at":"2026-08-18T11:43:22Z","signature_id":"539fbb23e688ee404324e7c6ef941c737fc7089dc370a96c3bb945e17dc41352","pr_number":6492,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-4984p9m7f6lc\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4984p9m7f6lc, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":13,"runs_affected":13,"failed_runs":13,"run_impact_percent":2.2146507666098807,"anchor_occurrences":13,"after_last_push_count":2,"also_in":["STG"],"trend_counts":[0,3,2,1,4,2,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":20,"prior_last_seen_at":"2026-08-22T14:52:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":9},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:208]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00396d0e0\u003e: \n    failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-4984p9m7f6lc\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4984p9m7f6lc, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:208]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00396d0e0\u003e: \n    failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-4984p9m7f6lc\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-4984p9m7f6lc, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:245]: failed to scale down node pool \"np-scale\" from 2 to 1 replicas\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002c86f00\u003e: \n    failed waiting for nodepool=\"np-scale\" in cluster=\"priv-np-scale\" resourcegroup=\"priv-np-scale-qtxpc74p5cw9\" to finish updating, caused by: timeout '60.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-scale in cluster priv-np-scale in resource group priv-np-scale-qtxpc74p5cw9, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:245]: failed to scale down node pool \"np-scale\" from 2 to 1 replicas\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002c86f00\u003e: \n    failed waiting for nodepool=\"np-scale\" in cluster=\"priv-np-scale\" resourcegroup=\"priv-np-scale-qtxpc74p5cw9\" to finish updating, caused by: timeout '60.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-scale in cluster priv-np-scale in resource group priv-np-scale-qtxpc74p5cw9, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:245]: failed to scale down node pool \"np-scale\" from 2 to 1 replicas\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011d67b0\u003e: \n    failed waiting for nodepool=\"np-scale\" in cluster=\"priv-np-scale\" resourcegroup=\"priv-np-scale-fcxcqdz2d2x2\" to finish updating, caused by: timeout '40.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-scale in cluster priv-np-scale in resource group priv-np-scale-fcxcqdz2d2x2, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:245]: failed to scale down node pool \"np-scale\" from 2 to 1 replicas\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011d67b0\u003e: \n    failed waiting for nodepool=\"np-scale\" in cluster=\"priv-np-scale\" resourcegroup=\"priv-np-scale-fcxcqdz2d2x2\" to finish updating, caused by: timeout '40.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-scale in cluster priv-np-scale in resource group priv-np-scale-fcxcqdz2d2x2, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091855258673221632","occurred_at":"2026-08-24T11:48:53Z","signature_id":"dfb206f2ba6bc7bc3ce13c32e974065647bae3407bafdce87be1e26f94b6936e","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091176636450869248","occurred_at":"2026-08-22T14:52:16Z","signature_id":"2279f0e21f0118ec4cfb855020d1fb23afa7a8053b7e28a26a3e788f00eff3ba","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090989679758282752","occurred_at":"2026-08-22T02:29:22Z","signature_id":"79acbfdba47e0e4a6c76392e334f88cd638d51646a498b2bfb66f8deaf94d4c5","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090884661910704128","occurred_at":"2026-08-21T19:32:04Z","signature_id":"02f8cf971ee2086a5a5c0f3e34976f3baeeca58a897d042c67c691ba1f89b42d","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090833794218070016","occurred_at":"2026-08-21T16:09:56Z","signature_id":"031dc05be95ed92f826a45611c2e85f662a26b57a5843c7a19763beaeca101c2","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090817534927310848","occurred_at":"2026-08-21T15:05:19Z","signature_id":"d3641496165ea9fc11207fde332bc727e7add4426bbba9ed882ee06f8b3ed6aa","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090766839033368576","occurred_at":"2026-08-21T11:43:53Z","signature_id":"ce181594deb79c253e35d008cf1bd9fa400dffdb87c80b1827c3a6d4e05ad422","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090355263239360512","occurred_at":"2026-08-20T08:28:26Z","signature_id":"10a5ec416a1194f12713357f6f75e0d897f7d7fbfc371d611a8f593beff77b00","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090181452632166400","occurred_at":"2026-08-19T20:57:46Z","signature_id":"5cf10238ae32cb119c8dedd783e284c2ab85fb7d2d666794b8ad8ff1902d5a2c","pr_number":6572,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"67a86f16b7c04eb1bdd0def775c5fa85beab5151539ba43c3c263ef5583b993e","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6589/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089793767912509440","occurred_at":"2026-08-18T19:17:13Z","signature_id":"7cd618739b4250f08ac47045f1a8f4cfaefb9d2ba149d13894eaf60f24468614","pr_number":6589,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"15d09de4ace49805af7392a55011596a42756f01603ad1dbadc2bd03375150c2","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089644114403397632","occurred_at":"2026-08-18T09:22:35Z","signature_id":"508230ce06c6aa7065ffe6698f28b6453a152fccc32431a8814b9e8c8618b0cc","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"50e9a505dfd78e0f2ab37ff9c69326a4b56cb21d9921b84b1255256ca501391e","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":13,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.5110732538330494,"anchor_occurrences":13,"after_last_push_count":0,"trend_counts":[0,8,5,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-19T14:06:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00145c120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-fbtznfmgpfmh\" to finish deleting: GET https://rp.j0627968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/823db68c-79dc-4727-9121-95296501f4bf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/823db68c-79dc-4727-9121-95296501f4bf\",\n      \"name\": \"823db68c-79dc-4727-9121-95296501f4bf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:01:32.394014466Z\",\n      \"endTime\": \"2026-08-19T15:25:40.293432829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9n9jdesahjtqdjvpc60o8amh8beqdt still exists (deletion dispatched at 2026-08-19T15:01:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 11 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00145c120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-fbtznfmgpfmh\" to finish deleting: GET https://rp.j0627968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/823db68c-79dc-4727-9121-95296501f4bf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/823db68c-79dc-4727-9121-95296501f4bf\",\n      \"name\": \"823db68c-79dc-4727-9121-95296501f4bf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:01:32.394014466Z\",\n      \"endTime\": \"2026-08-19T15:25:40.293432829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9n9jdesahjtqdjvpc60o8amh8beqdt still exists (deletion dispatched at 2026-08-19T15:01:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 11 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011603c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-d9djl4vwvd9d\" to finish deleting: GET https://rp.j0627968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1922b32-d944-4e67-9298-3423ac7d756d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1922b32-d944-4e67-9298-3423ac7d756d\",\n      \"name\": \"a1922b32-d944-4e67-9298-3423ac7d756d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:21:24.495403431Z\",\n      \"endTime\": \"2026-08-19T15:45:30.286773534Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9n9jldsfibh07akcms9kk0t1s8lje8 still exists (deletion dispatched at 2026-08-19T15:21:26Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 13 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011603c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-d9djl4vwvd9d\" to finish deleting: GET https://rp.j0627968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1922b32-d944-4e67-9298-3423ac7d756d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1922b32-d944-4e67-9298-3423ac7d756d\",\n      \"name\": \"a1922b32-d944-4e67-9298-3423ac7d756d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:21:24.495403431Z\",\n      \"endTime\": \"2026-08-19T15:45:30.286773534Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9n9jldsfibh07akcms9kk0t1s8lje8 still exists (deletion dispatched at 2026-08-19T15:21:26Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 13 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011fc078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-6hk7rf89rgds\" to finish deleting: GET https://rp.j0627968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52d6a689-7624-4713-8e33-581b49068dde\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52d6a689-7624-4713-8e33-581b49068dde\",\n      \"name\": \"52d6a689-7624-4713-8e33-581b49068dde\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:00:27.665748341Z\",\n      \"endTime\": \"2026-08-19T15:24:30.376504783Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9n9ia6c2epu24n3nrjgvgsrmalvbl6 still exists (deletion dispatched at 2026-08-19T15:00:28Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 11 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011fc078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-6hk7rf89rgds\" to finish deleting: GET https://rp.j0627968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52d6a689-7624-4713-8e33-581b49068dde\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/52d6a689-7624-4713-8e33-581b49068dde\",\n      \"name\": \"52d6a689-7624-4713-8e33-581b49068dde\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:00:27.665748341Z\",\n      \"endTime\": \"2026-08-19T15:24:30.376504783Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9n9ia6c2epu24n3nrjgvgsrmalvbl6 still exists (deletion dispatched at 2026-08-19T15:00:28Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 11 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"1a7c8d9caa8688de4a4ef04fd863ee389fb3f378d9c8a57e5ad29e4240c26533","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"7dd2e6e13f18d3730b3936e8c95ad3dab0591a65b4972dd217ed46e1d47252fe","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"859c1c841e7e068411598a3f82f1d73f33865edd7a233108aac62d1ee6597008","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"8cfdc425aef7a589789ed69f171d6c8db0c98e46bb0da973215ed67927b6011a","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"fcdd64be3e6d3a09882e5be2f0fca23526c058bbbcf10d8bc5931108751b3ba4","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"2d4af2e8818b6ec3f4694c6bbaa5060f39345860fc22cc4d87ddc87577515adb","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"5ff7760b4fc71db53052ad67441e0f75ac1798eaaa39855d1ffb8337c1a1b8e9","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"da33b2efa410279d5ecaf2e09f11306fe43e9aad62a65a7b0dc813537b7b689e","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"f2122ecb2380713532acde2f740a203e4def31892958008312ef4d9bd32491ff","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"febf954cb8d1010002aed95c5483f9c49bce15823edda4b680d02cde1f5b452c","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089644114403397632","occurred_at":"2026-08-18T09:22:35Z","signature_id":"458016ff58df480a9af890efc4869cd281a68016880f7660f3cb3caa86d761ee","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089644114403397632","occurred_at":"2026-08-18T09:22:35Z","signature_id":"cb74463b34c7c5e058321c6e246a15fc19fa5d16a841f156bfcd8b8e4edcb424","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089644114403397632","occurred_at":"2026-08-18T09:22:35Z","signature_id":"f44c67510eeadd4b81caa54f90f612ee5cc0165c83f7c5c639bba29933f36275","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f3617ed13d3ab451028cceb172cac88201779eed87db517d9d9d4307cf6c5d4a","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":12,"runs_affected":10,"failed_runs":10,"run_impact_percent":1.7035775127768313,"anchor_occurrences":12,"after_last_push_count":9,"also_in":["PROD","STG"],"trend_counts":[0,3,2,1,1,1,0,4],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":12,"prior_last_seen_at":"2026-08-22T00:10:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00123e240\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2xcnpkr48kxk\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3185d8c9-e246-4082-b170-898da28192d1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3185d8c9-e246-4082-b170-898da28192d1\",\n      \"name\": \"3185d8c9-e246-4082-b170-898da28192d1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T17:28:32.888311428Z\",\n      \"endTime\": \"2026-08-24T17:47:34.534131157Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00123e240\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2xcnpkr48kxk\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3185d8c9-e246-4082-b170-898da28192d1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3185d8c9-e246-4082-b170-898da28192d1\",\n      \"name\": \"3185d8c9-e246-4082-b170-898da28192d1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T17:28:32.888311428Z\",\n      \"endTime\": \"2026-08-24T17:47:34.534131157Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:726]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dd0320\u003e: \n    failed to create HCP cluster np-dg-minor-4tds4q: failed waiting for cluster=\"np-dg-minor-4tds4q\" in resourcegroup=\"rg-np-dg-minor-4tds4q-6qtw9l2fwcdn\" to finish creating: GET https://rp.j8539776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/162a10b5-886b-40a8-a95b-30fc38ce9bc5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/162a10b5-886b-40a8-a95b-30fc38ce9bc5\",\n      \"name\": \"162a10b5-886b-40a8-a95b-30fc38ce9bc5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:41:41.593852701Z\",\n      \"endTime\": \"2026-08-24T16:00:49.106385261Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:726]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dd0320\u003e: \n    failed to create HCP cluster np-dg-minor-4tds4q: failed waiting for cluster=\"np-dg-minor-4tds4q\" in resourcegroup=\"rg-np-dg-minor-4tds4q-6qtw9l2fwcdn\" to finish creating: GET https://rp.j8539776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/162a10b5-886b-40a8-a95b-30fc38ce9bc5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/162a10b5-886b-40a8-a95b-30fc38ce9bc5\",\n      \"name\": \"162a10b5-886b-40a8-a95b-30fc38ce9bc5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:41:41.593852701Z\",\n      \"endTime\": \"2026-08-24T16:00:49.106385261Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e96560\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-fn2m5nmfttf7\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7b7e77-c07f-48f4-a1bf-2f48bcb6434a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7b7e77-c07f-48f4-a1bf-2f48bcb6434a\",\n      \"name\": \"eb7b7e77-c07f-48f4-a1bf-2f48bcb6434a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:17:08.176674236Z\",\n      \"endTime\": \"2026-08-24T15:36:17.96458443Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e96560\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-fn2m5nmfttf7\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7b7e77-c07f-48f4-a1bf-2f48bcb6434a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7b7e77-c07f-48f4-a1bf-2f48bcb6434a\",\n      \"name\": \"eb7b7e77-c07f-48f4-a1bf-2f48bcb6434a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:17:08.176674236Z\",\n      \"endTime\": \"2026-08-24T15:36:17.96458443Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"e79e05105a6dd37d39aa319e6b82dc1c85c13b7322507d230499b9d98521e94c","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091898914398539776","occurred_at":"2026-08-24T14:42:21Z","signature_id":"7f0730f0d1c7e0eb7a035ce2e1a824a7fb99f16bf281917ec8e0ea592b939bdd","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"f45d8f4fc578e9bcbc89c15b0ea557b08d4323587837d33c561c269a0b539be9","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895460682797056","occurred_at":"2026-08-24T14:28:36Z","signature_id":"f749c71c85c572aa2c57394be8a931e71c30eda717753e81fa6456d997c9c5fc","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090954745601593344","occurred_at":"2026-08-22T00:10:34Z","signature_id":"023f4e4e9644f027df72df49213653e5102b64065aaeeb7d55a65b696305b4e0","pr_number":6593,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090643626764800000","occurred_at":"2026-08-21T03:34:17Z","signature_id":"417af16cda9f0be1e205be17ce85652108e7367e103bf854b8e12c7538a97141","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090314560790597632","occurred_at":"2026-08-20T05:46:42Z","signature_id":"ec821af0caf46bb7fcb64607776a1ac9117bee65d538374ed623769ca4db28c1","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"72ebafea3e11c609d2f9a462699160dab0bb18b50d4ec6bd66dc45358da28bbd","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"d8848b082a36fe7aa93b5aadfb34fc3693bade5b697b59af38856412db575df6","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"6774fcda32123ac6f2680d3a613c63cdbdecb7ead09d7395032f9742643cb4bd","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"a93d3d94ae9042d91e7bd42d6a192960f43bb08a57ae75979c494d5edd463089","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089723779814002688","occurred_at":"2026-08-18T14:39:04Z","signature_id":"16f0252e3240586a314eda9683d3f15adf1f0e584366777e9c58653ed1edb220","pr_number":6157,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a871b3ae1e8360285972402ef71b07429b5d17f69841894ee06b053d348eb8de","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/management/output (stamp=1)","occurrences":11,"runs_affected":11,"failed_runs":11,"run_impact_percent":1.8739352640545144,"anchor_occurrences":11,"after_last_push_count":10,"trend_counts":[5,3,1,1,4,0,0,2],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":27,"prior_last_seen_at":"2026-08-21T16:05:36Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/management/output (stamp=1)","occurrences":6},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/management/output (stamp=1)","occurrences":5}],"full_error_samples":["time=2026-08-24T18:57:37.594Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=output stamp=1 description=\"Step output\\n  Kind: ARM\\n  Template: ./../dev-infrastructure/templates/output-mgmt.bicep\\n  Parameters: ./../dev-infrastructure/configurations/output-mgmt.tmpl.bicepparam\"\ntime=2026-08-24T18:57:37.979Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=output stamp=1\ntime=2026-08-24T18:57:38.350Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=output stamp=1 deployment=cfd650f4557893b92396842785417ac4f900952cc40f35ef046b64198c66a837 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8797056-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fcfd650f4557893b92396842785417ac4f900952cc40f35ef046b64198c66a837\ntime=2026-08-24T18:57:38.400Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=output stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8797056-mgmt-1/providers/Microsoft.Resources/deployments/cfd650f4557893b92396842785417ac4f900952cc40f35ef046b64198c66a837/operationStatuses/08584140090272578417\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentNotFound\\\",\\n    \\\"message\\\": \\\"Deployment 'cfd650f4557893b92396842785417ac4f900952cc40f35ef046b64198c66a837' could not be found.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-08-24T17:20:28.034Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 description=\"Step output\\n  Kind: ARM\\n  Template: ../../dev-infrastructure/templates/output-mgmt.bicep\\n  Parameters: ../../dev-infrastructure/configurations/output-mgmt.tmpl.bicepparam\"\ntime=2026-08-24T17:20:28.426Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1\ntime=2026-08-24T17:20:28.817Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 deployment=3472b226dd1bf3f5c2c2fbd8f62937b0fe2f97cb4ca8a63168999f9ba0945d23 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2076032-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F3472b226dd1bf3f5c2c2fbd8f62937b0fe2f97cb4ca8a63168999f9ba0945d23\ntime=2026-08-24T17:20:28.866Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2076032-mgmt-1/providers/Microsoft.Resources/deployments/3472b226dd1bf3f5c2c2fbd8f62937b0fe2f97cb4ca8a63168999f9ba0945d23/operationStatuses/08584140148567905734\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentNotFound\\\",\\n    \\\"message\\\": \\\"Deployment '3472b226dd1bf3f5c2c2fbd8f62937b0fe2f97cb4ca8a63168999f9ba0945d23' could not be found.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-08-21T16:46:53.918Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=output stamp=1 description=\"Step output\\n  Kind: ARM\\n  Template: ./../dev-infrastructure/templates/output-mgmt.bicep\\n  Parameters: ./../dev-infrastructure/configurations/output-mgmt.tmpl.bicepparam\"\ntime=2026-08-21T16:46:54.412Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=output stamp=1\ntime=2026-08-21T16:46:54.927Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=output stamp=1 deployment=d8863692838cbe3c0e6e8e47c2c166e7f2539a854071effa428b1dfa89818a4d portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0032384-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fd8863692838cbe3c0e6e8e47c2c166e7f2539a854071effa428b1dfa89818a4d\ntime=2026-08-21T16:46:55.024Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=output stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0032384-mgmt-1/providers/Microsoft.Resources/deployments/d8863692838cbe3c0e6e8e47c2c166e7f2539a854071effa428b1dfa89818a4d/operationStatuses/08584142760707149990\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentNotFound\\\",\\n    \\\"message\\\": \\\"Deployment 'd8863692838cbe3c0e6e8e47c2c166e7f2539a854071effa428b1dfa89818a4d' could not be found.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091954836298797056","occurred_at":"2026-08-24T18:15:46Z","signature_id":"0ec7efd378b135d7fa85cb93bcc2f610fe36f768df0ac4e2e0fe6122f4092f85","pr_number":6504,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928527292076032","occurred_at":"2026-08-24T16:32:12Z","signature_id":"786d454f94fa948d60967288b990bcecab5b5254cf35274ddf07e63a4edbe1f5","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6650/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090832700310032384","occurred_at":"2026-08-21T16:05:36Z","signature_id":"0ec7efd378b135d7fa85cb93bcc2f610fe36f768df0ac4e2e0fe6122f4092f85","pr_number":6650,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090730615539240960","occurred_at":"2026-08-21T09:19:57Z","signature_id":"786d454f94fa948d60967288b990bcecab5b5254cf35274ddf07e63a4edbe1f5","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090717629881782272","occurred_at":"2026-08-21T08:28:21Z","signature_id":"786d454f94fa948d60967288b990bcecab5b5254cf35274ddf07e63a4edbe1f5","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090618928278540288","occurred_at":"2026-08-21T01:56:08Z","signature_id":"0ec7efd378b135d7fa85cb93bcc2f610fe36f768df0ac4e2e0fe6122f4092f85","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090541078221426688","occurred_at":"2026-08-20T20:46:48Z","signature_id":"786d454f94fa948d60967288b990bcecab5b5254cf35274ddf07e63a4edbe1f5","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6515/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090152370880122880","occurred_at":"2026-08-19T19:02:11Z","signature_id":"786d454f94fa948d60967288b990bcecab5b5254cf35274ddf07e63a4edbe1f5","pr_number":6515,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6527/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089795500311056384","occurred_at":"2026-08-18T19:24:08Z","signature_id":"0ec7efd378b135d7fa85cb93bcc2f610fe36f768df0ac4e2e0fe6122f4092f85","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089787112214761472","occurred_at":"2026-08-18T18:49:18Z","signature_id":"0ec7efd378b135d7fa85cb93bcc2f610fe36f768df0ac4e2e0fe6122f4092f85","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089750086291558400","occurred_at":"2026-08-18T16:23:40Z","signature_id":"786d454f94fa948d60967288b990bcecab5b5254cf35274ddf07e63a4edbe1f5","pr_number":6573,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6cf0bbf9b7d17f41cb34ef56e829ec2324222ca77c76bbfc427185392db35615","failure_pattern":"Error from server (ServiceUnavailable): the server is currently unable to handle the request)","search_query":"Error from server (ServiceUnavailable): the server is currently unable to handle the request)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS and private ingress and verify both are only reachable from VNet","occurrences":11,"runs_affected":6,"failed_runs":6,"run_impact_percent":1.0221465076660987,"anchor_occurrences":11,"after_last_push_count":0,"trend_counts":[0,0,0,0,3,6,2,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-23T03:43:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS and private ingress and verify both are only reachable from VNet","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS, private ingress, and private KeyVault and verify all are operational","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:172]: Timed out after 3623.396s.\nKAS should be reachable from VM via internal LB\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:170 with:\nkubectl version should show Server Version, proving KAS is reachable from VM (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nError from server (ServiceUnavailable): the server is currently unable to handle the request)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:172]: Timed out after 3623.396s.\nKAS should be reachable from VM via internal LB\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:170 with:\nkubectl version should show Server Version, proving KAS is reachable from VM (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nError from server (ServiceUnavailable): the server is currently unable to handle the request)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:203]: Timed out after 3630.452s.\nKAS should be reachable from VM via internal LB\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:201 with:\nkubectl version should show Server Version, proving KAS is reachable from VM (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nError from server (ServiceUnavailable): the server is currently unable to handle the request)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:203]: Timed out after 3630.452s.\nKAS should be reachable from VM via internal LB\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:201 with:\nkubectl version should show Server Version, proving KAS is reachable from VM (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nError from server (ServiceUnavailable): the server is currently unable to handle the request)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:172]: Timed out after 1221.685s.\nKAS should be reachable from VM via internal LB\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:170 with:\nkubectl version should show Server Version, proving KAS is reachable from VM (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nError from server (ServiceUnavailable): the server is currently unable to handle the request)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:172]: Timed out after 1221.685s.\nKAS should be reachable from VM via internal LB\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:170 with:\nkubectl version should show Server Version, proving KAS is reachable from VM (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nError from server (ServiceUnavailable): the server is currently unable to handle the request)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091370674797940736","occurred_at":"2026-08-23T03:43:19Z","signature_id":"a2d132dfde07f3391ff7b4a1c2de45cb2c603d801bc82cc4a5fbbe7df390ef7e","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091370674797940736","occurred_at":"2026-08-23T03:43:19Z","signature_id":"d2285d363ed1a1e560ce21be276ca5a6521530a681d4f19d7cb7f03800b0b0a1","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091230843602210816","occurred_at":"2026-08-22T18:27:40Z","signature_id":"b7d765b338677f9132d9cff7a564a334e39b7d73a284b7fd67bdad2484900a37","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091230843602210816","occurred_at":"2026-08-22T18:27:40Z","signature_id":"e93aca990e9358976e512f671aa4caa5d2bcaeab27afcaa4ca5fdcddc64ad356","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091176636450869248","occurred_at":"2026-08-22T14:52:16Z","signature_id":"69c3f7216d3016ebb9d2fb04a005788dacee246d0f78bb02875390b4e0edcae0","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091176636450869248","occurred_at":"2026-08-22T14:52:16Z","signature_id":"d8865b61f871581c8d18c544a842a91b67cbd9285ac45828a068750fdf612811","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090989679758282752","occurred_at":"2026-08-22T02:29:22Z","signature_id":"3822fa1effd49ba7c1cdd3d0033f308a6711f286fd0ceab8d0d8b7b1e0be414a","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090989679758282752","occurred_at":"2026-08-22T02:29:22Z","signature_id":"d5ea982e91fda6f482df8e6cf7fbadfda4c5f9261aef6f580c5e75ddca0ed96d","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090930773002555392","occurred_at":"2026-08-21T22:35:18Z","signature_id":"d5ea982e91fda6f482df8e6cf7fbadfda4c5f9261aef6f580c5e75ddca0ed96d","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090884661910704128","occurred_at":"2026-08-21T19:32:04Z","signature_id":"b3f9577b121e1d710a77d5ca9c61cd869d4fd50ea68e082da03b5517331dd8cd","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090884661910704128","occurred_at":"2026-08-21T19:32:04Z","signature_id":"b3f9577b121e1d710a77d5ca9c61cd869d4fd50ea68e082da03b5517331dd8cd","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d3766fe34f03f22876c777e51a7be1734ee728533bfb3f28e111bf0b35fcad1b","failure_pattern":"UpdateHCPCluster20260630 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-cflktt657pch\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-cflktt657pch, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":10,"runs_affected":10,"failed_runs":10,"run_impact_percent":1.7035775127768313,"anchor_occurrences":10,"after_last_push_count":2,"trend_counts":[0,3,3,1,2,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":60,"prior_last_seen_at":"2026-08-21T15:05:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":10}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:172]: failed to update cluster with new KMS key\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00132c390\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-cflktt657pch\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-cflktt657pch, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:172]: failed to update cluster with new KMS key\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00132c390\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-cflktt657pch\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-cflktt657pch, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:172]: failed to update cluster with new KMS key\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00154c060\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-hhjbvcghxvgq\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-hhjbvcghxvgq, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:172]: failed to update cluster with new KMS key\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00154c060\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-hhjbvcghxvgq\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-hhjbvcghxvgq, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:172]: failed to update cluster with new KMS key\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f7e900\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-ntm46dnmmclw\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-ntm46dnmmclw, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:172]: failed to update cluster with new KMS key\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f7e900\u003e: \n    failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-ntm46dnmmclw\" to finish updating, caused by: timeout '30.000000' minutes exceeded during UpdateHCPCluster20260630 for cluster kms-key-rotate-422 in resource group kms-key-rotate-ntm46dnmmclw, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091855258673221632","occurred_at":"2026-08-24T11:48:53Z","signature_id":"78869f45f7134e749985fc5b23dff71b456538a41d70734a716f1e608670ab6c","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090817534927310848","occurred_at":"2026-08-21T15:05:19Z","signature_id":"f8c1f9d6e4a842b1616d9cf1c6c5f32baae9995231c4a073b145ef14db36466d","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090766839033368576","occurred_at":"2026-08-21T11:43:53Z","signature_id":"73c65419fc618c891d660bbf76f7851fce8c4689b50628554d1a8c94e626ef6a","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"f4dcc26904b6619c31c3833aeb00b8bdbc202a180a5f1e9f30f427ca0234c47d","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090126361980047360","occurred_at":"2026-08-19T17:18:51Z","signature_id":"fb4032a22ac181febfd936b28ac5ea4e27236ca4b44c9cca6e758fa816729066","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090079394612645888","occurred_at":"2026-08-19T14:12:08Z","signature_id":"7c9a82bf8c17109250ce7aecb9c10e968aa4ef37dcbf68fe936e3a7e20e58b13","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"3eebc351608e267dcac0b25afc008409f4090bfbafdbe3d7c34eb3432bd2f54e","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"4706928bc92818dd10f6d01c19a730ee57c57513f42059ae660b4acd3ffaf3dc","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089651610870353920","occurred_at":"2026-08-18T09:52:22Z","signature_id":"0e669e261d1988e6d7625695608ffac20e51e62db0475e398e71418685ac4c03","pr_number":6562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089644114403397632","occurred_at":"2026-08-18T09:22:35Z","signature_id":"94fb75c74dab603d03044dfc60878332b33b05f844d2318dc85b21669c63e9b2","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":10,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.6814310051107325,"anchor_occurrences":10,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,10],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-22lvs576ptrb/cluster-candidate-4-20-qdcf69 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001190ba0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-qdcf69: failed waiting for cluster=\"cluster-candidate-4-20-qdcf69\" in resourcegroup=\"rg-candidate-4-20-22lvs576ptrb\" to finish creating: GET https://rp.j9409536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dc1dd563-6c7e-4d20-8fce-99e12ec22770\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dc1dd563-6c7e-4d20-8fce-99e12ec22770\",\n      \"name\": \"dc1dd563-6c7e-4d20-8fce-99e12ec22770\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:51:04.562343833Z\",\n      \"endTime\": \"2026-08-24T22:10:06.652345811Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-22lvs576ptrb/cluster-candidate-4-20-qdcf69 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001190ba0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-qdcf69: failed waiting for cluster=\"cluster-candidate-4-20-qdcf69\" in resourcegroup=\"rg-candidate-4-20-22lvs576ptrb\" to finish creating: GET https://rp.j9409536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dc1dd563-6c7e-4d20-8fce-99e12ec22770\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dc1dd563-6c7e-4d20-8fce-99e12ec22770\",\n      \"name\": \"dc1dd563-6c7e-4d20-8fce-99e12ec22770\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:51:04.562343833Z\",\n      \"endTime\": \"2026-08-24T22:10:06.652345811Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e33220\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-ndmwcdqgk685\" to finish creating: GET https://rp.j9409536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a097060e-673b-40de-a82a-aa623235d88d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a097060e-673b-40de-a82a-aa623235d88d\",\n      \"name\": \"a097060e-673b-40de-a82a-aa623235d88d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:51:06.650394368Z\",\n      \"endTime\": \"2026-08-24T22:10:06.638990375Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e33220\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-ndmwcdqgk685\" to finish creating: GET https://rp.j9409536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a097060e-673b-40de-a82a-aa623235d88d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a097060e-673b-40de-a82a-aa623235d88d\",\n      \"name\": \"a097060e-673b-40de-a82a-aa623235d88d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:51:06.650394368Z\",\n      \"endTime\": \"2026-08-24T22:10:06.638990375Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:148]: failed to create HCP cluster np-version-upgrade-cluster-vkwj8z with version 4.21.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bca9c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-vkwj8z: failed waiting for cluster=\"np-version-upgrade-cluster-vkwj8z\" in resourcegroup=\"rg-np-version-upgrade-vkwj8z-pvn2kjtfhwm4\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ca593c6-172c-442e-bdb4-2b96ae721c04\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ca593c6-172c-442e-bdb4-2b96ae721c04\",\n      \"name\": \"9ca593c6-172c-442e-bdb4-2b96ae721c04\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:29:02.41159338Z\",\n      \"endTime\": \"2026-08-24T21:48:06.346460728Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:148]: failed to create HCP cluster np-version-upgrade-cluster-vkwj8z with version 4.21.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bca9c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-vkwj8z: failed waiting for cluster=\"np-version-upgrade-cluster-vkwj8z\" in resourcegroup=\"rg-np-version-upgrade-vkwj8z-pvn2kjtfhwm4\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ca593c6-172c-442e-bdb4-2b96ae721c04\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ca593c6-172c-442e-bdb4-2b96ae721c04\",\n      \"name\": \"9ca593c6-172c-442e-bdb4-2b96ae721c04\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:29:02.41159338Z\",\n      \"endTime\": \"2026-08-24T21:48:06.346460728Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"87055b770927de3226a00a72e6a1e8f425c4181b7305ba9fd94502a36df5fec7","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6660/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091994848889409536","occurred_at":"2026-08-24T21:03:33Z","signature_id":"e5919a5aa6e234b2c6b16539e4375d2ff9f7724d31d00cb9d8f2c6f3bd3b38ad","pr_number":6660,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"4bba4cf482e3fa39dfa2ce3e799d7eb2c477fcc7e6ad0b0350124af0d131c053","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"5a1d95e1ce2afb3cd1318374aea17f4a6e4f5d30ef03f9f5a0cc441e9043a771","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"28a8b869f35a9cd1794a8e197fc2255f15a11da6d73b85987e232efa1871d3dc","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"2cae48dc9a7ac77e5a8ac54a32e19e04d69dc07b4bbfdffac675fbb83525b59a","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"51d9d625d26042275bccf29b990ed16b6ab0c82bab381da48ae1943a65da776a","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"57e93beced19ad75ed8576f3f0efd935e4fd8431d2ed07db01840f4e2fefa928","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"9137a39c80626f35b65cbd5b530df9d8f5ceed8a3ed523d23d20fe02b5217e4f","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091899442289446912","occurred_at":"2026-08-24T14:44:27Z","signature_id":"0119ef26d501844a6f69a87b74e13b68dec77db9c3af742f0d6e0ca5c74e1853","pr_number":6629,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":9,"runs_affected":9,"failed_runs":9,"run_impact_percent":1.5332197614991483,"anchor_occurrences":9,"after_last_push_count":5,"trend_counts":[0,2,2,2,0,2,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":35,"prior_last_seen_at":"2026-08-22T08:48:46Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":9}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T18:02:20Z\n  Ended:       2026-08-24T18:38:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4cebd923-31c4-4c0a-b568-63bf1a281b9a\"\n  Description: Pod ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T18:03:20Z\n  Ended:       2026-08-24T18:38:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/hcps-j0375808\", namespace=\"ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4cebd923-31c4-4c0a-b568-63bf1a281b9a\"\n  Description: Pod ocm-arohcpci01-2sd37mkptl2qprba4ejkvlbldgl77u9k-r3i6h6d2n5v5b4i/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-22T09:53:49Z\n  Ended:       2026-08-22T10:12:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3442688-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.129:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3442688/providers/microsoft.monitor/accounts/hcps-j3442688\", namespace=\"ocm-arohcpci01-2sbhtcj4djmllishlrj1ubeicnu0g5pf-sre-hcp-cluster\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"73b11fbd-18cc-490d-bc49-f492731b9079\"\n  Description: Pod ocm-arohcpci01-2sbhtcj4djmllishlrj1ubeicnu0g5pf-sre-hcp-cluster/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-22T09:54:46Z\n  Ended:       2026-08-22T10:12:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3442688-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.129:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3442688/providers/microsoft.monitor/accounts/hcps-j3442688\", namespace=\"ocm-arohcpci01-2sbhtcj4djmllishlrj1ubeicnu0g5pf-sre-hcp-cluster\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"73b11fbd-18cc-490d-bc49-f492731b9079\"\n  Description: Pod ocm-arohcpci01-2sbhtcj4djmllishlrj1ubeicnu0g5pf-sre-hcp-cluster/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-22T01:46:18Z\n  Ended:       2026-08-22T01:59:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2714112-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.77:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714112/providers/microsoft.monitor/accounts/hcps-j2714112\", namespace=\"ocm-arohcpci01-2sbapa1q98vqjgeok5d2vk39oquh04bn-e1r3y8g6m6m5y1i\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"2f5f22c6-a50a-4fc1-856c-91f66f6f3ed2\"\n  Description: Pod ocm-arohcpci01-2sbapa1q98vqjgeok5d2vk39oquh04bn-e1r3y8g6m6m5y1i/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-22T01:46:18Z\n  Ended:       2026-08-22T02:06:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2714112-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.77:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714112/providers/microsoft.monitor/accounts/hcps-j2714112\", namespace=\"ocm-arohcpci01-2sbapa1q98vqjgeok5d2vk39oquh04bn-e1r3y8g6m6m5y1i\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"2f5f22c6-a50a-4fc1-856c-91f66f6f3ed2\"\n  Description: Pod ocm-arohcpci01-2sbapa1q98vqjgeok5d2vk39oquh04bn-e1r3y8g6m6m5y1i/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"9d5e8ec654bc2ebd399a25878bada85329c8658827f68d63b0042fb7498d8047","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091085161763442688","occurred_at":"2026-08-22T08:48:46Z","signature_id":"7c60f219c222a62d33851943284ebbb72e284ca27ae434afbd2eb93ae7311af5","pr_number":6640,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090960143322714112","occurred_at":"2026-08-22T00:32:00Z","signature_id":"67afc4691e036888d68d3d1a097dff27f3dee600ac245480c61c302a28649421","pr_number":6639,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090465920358027264","occurred_at":"2026-08-20T15:48:08Z","signature_id":"2e5d452b37114ba6800554f43052dc62e06de28d819190587202da2dd8d76bd2","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090405439429677056","occurred_at":"2026-08-20T11:47:38Z","signature_id":"ff1b491f2f616f8c76c519fe7e81f5344a08bdc867b275f3ca98624a57a64102","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090191599463043072","occurred_at":"2026-08-19T21:38:05Z","signature_id":"beaf2c9e6ccf7a0afb381b01f5e95b286fbc41966ac0bd4afa4af24042def984","pr_number":6608,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090133175878553600","occurred_at":"2026-08-19T17:45:52Z","signature_id":"0a1f68d8ba1568ae18db579b0b5ca5f8a090856f14ce7ebf1a338ca265ecc142","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089754858566455296","occurred_at":"2026-08-18T16:42:38Z","signature_id":"fe0e181f61512742c5adabdc0518af3aab5d21d78205d6935da05a4e1659d121","pr_number":6553,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089753382901256192","occurred_at":"2026-08-18T16:36:46Z","signature_id":"c866b972d4f53e2c3ec306928793da45d871ff907b10275ef5997aad904a0cbb","pr_number":5900,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":9,"runs_affected":9,"failed_runs":9,"run_impact_percent":1.5332197614991483,"anchor_occurrences":9,"after_last_push_count":5,"trend_counts":[2,1,3,2,2,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":51,"prior_last_seen_at":"2026-08-21T05:44:12Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":9}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T14:40:10Z\n  Ended:       2026-08-24T14:49:11Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6438784-svc\", component=\"backend\", name=\"csstatedump\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-21T07:30:21Z\n  Ended:       2026-08-21T07:38:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4553984-svc\", component=\"backend\", name=\"csstatedump\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-21T07:31:20Z\n  Ended:       2026-08-21T07:38:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4553984-svc\", component=\"backend\", name=\"datadump\", severity=\"warning\"\n  Description: Backend controller workqueue datadump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 3 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-21T05:41:29Z\n  Ended:       2026-08-21T05:56:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8412928-svc\", component=\"backend\", name=\"createclusterscopedreaddesires\", severity=\"warning\"\n  Description: Backend controller workqueue createclusterscopedreaddesires has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-21T05:41:29Z\n  Ended:       2026-08-21T05:56:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8412928-svc\", component=\"backend\", name=\"billingdump\", severity=\"warning\"\n  Description: Backend controller workqueue billingdump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 3:\n  State:       Resolved\n  Started:     2026-08-21T05:41:29Z\n  Ended:       2026-08-21T05:56:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8412928-svc\", component=\"backend\", name=\"datadump\", severity=\"warning\"\n  Description: Backend controller workqueue datadump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879927736438784","occurred_at":"2026-08-24T13:26:52Z","signature_id":"f42ba6cef1b20012612134284fc91db6cdee01f2a97291f342457df1df0028e6","pr_number":6433,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090676319984553984","occurred_at":"2026-08-21T05:44:12Z","signature_id":"a776dc9dd0c07bc72a2a24fcac7b5cf950c5185ee842e2520407e59933eb5fe6","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6363/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657643948412928","occurred_at":"2026-08-21T04:29:59Z","signature_id":"6df0be81a04ab8fcbefe9cec40b0a31944793bf2405e2b497fc22dcf57c31988","pr_number":6363,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6622/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090464934205853696","occurred_at":"2026-08-20T15:44:12Z","signature_id":"0650add4c8ee71c20554737cc9c8a8701b1ecdb4bb6417db99f807fb7975fcec","pr_number":6622,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6616/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090329417594179584","occurred_at":"2026-08-20T06:45:44Z","signature_id":"2045d121c82435792aa22041b6c6dbf2f0fd58077598faaae5237ca9ed5f314e","pr_number":6616,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090198433062195200","occurred_at":"2026-08-19T22:05:11Z","signature_id":"38a5c8e8a2f89b4898206fcbe6a9636f06fc0c2617db9418c6b5e3fb64c7bbfe","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090164198184062976","occurred_at":"2026-08-19T19:49:12Z","signature_id":"16ac7e70f1586d9f026debf16669d9c9370e34cf4b8d7dd6a2443ad454bba160","pr_number":6588,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090068841211105280","occurred_at":"2026-08-19T13:30:10Z","signature_id":"3a423c8780dda4ffb00c339498942ffdd379852403a9bf6d8301e5743d8d3aa2","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089677306816630784","occurred_at":"2026-08-18T11:34:26Z","signature_id":"add00a4144f55afa80ae9ad4a481c33ba3433339a629eb8b2d5768620b9e8a98","pr_number":6583,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":9,"runs_affected":9,"failed_runs":9,"run_impact_percent":1.5332197614991483,"anchor_occurrences":9,"after_last_push_count":9,"also_in":["PROD","STG"],"trend_counts":[3,1,1,0,0,1,0,6],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":54,"prior_last_seen_at":"2026-08-22T04:44:09Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":9}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091985911028912128","occurred_at":"2026-08-24T20:28:03Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"371e2dd52ef73c97161118e4d4efe23b9ace8422b8af743704455c3d7faec23b","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091899442289446912","occurred_at":"2026-08-24T14:44:27Z","signature_id":"ecd1663935e8debc79eea1bee9fef1a5b3f20d6f83b67bf343fec7e29acad10f","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"371e2dd52ef73c97161118e4d4efe23b9ace8422b8af743704455c3d7faec23b","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":6527,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f88ac713b613a590b2fe8fa69068a61ed7140b6fed609b9849143fda8fd9e327","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":9,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.8517887563884157,"anchor_occurrences":9,"after_last_push_count":0,"trend_counts":[0,0,6,3,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-20T03:19:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011680d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-z85t4l4vlbzm\" to finish deleting: GET https://rp.j0378240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b5056cdf-a481-4533-9ce3-097744d6b413\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b5056cdf-a481-4533-9ce3-097744d6b413\",\n      \"name\": \"b5056cdf-a481-4533-9ce3-097744d6b413\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T04:16:44.794311199Z\",\n      \"endTime\": \"2026-08-20T04:40:52.757543513Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011680d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-z85t4l4vlbzm\" to finish deleting: GET https://rp.j0378240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b5056cdf-a481-4533-9ce3-097744d6b413\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b5056cdf-a481-4533-9ce3-097744d6b413\",\n      \"name\": \"b5056cdf-a481-4533-9ce3-097744d6b413\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T04:16:44.794311199Z\",\n      \"endTime\": \"2026-08-20T04:40:52.757543513Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019f6078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-dvmg5tnhn7bl\" to finish deleting: GET https://rp.j0378240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/59a1bde3-3626-4d39-bb21-86f7656900a9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/59a1bde3-3626-4d39-bb21-86f7656900a9\",\n      \"name\": \"59a1bde3-3626-4d39-bb21-86f7656900a9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T04:15:36.007694317Z\",\n      \"endTime\": \"2026-08-20T04:39:42.78156064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019f6078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-dvmg5tnhn7bl\" to finish deleting: GET https://rp.j0378240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/59a1bde3-3626-4d39-bb21-86f7656900a9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/59a1bde3-3626-4d39-bb21-86f7656900a9\",\n      \"name\": \"59a1bde3-3626-4d39-bb21-86f7656900a9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T04:15:36.007694317Z\",\n      \"endTime\": \"2026-08-20T04:39:42.78156064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014d2078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-j9kfk6656slq\" to finish deleting: GET https://rp.j3139200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/758cadb0-636e-4143-b1b5-5771c89ce22a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/758cadb0-636e-4143-b1b5-5771c89ce22a\",\n      \"name\": \"758cadb0-636e-4143-b1b5-5771c89ce22a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T02:09:52.194794521Z\",\n      \"endTime\": \"2026-08-20T02:33:57.777012859Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014d2078\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-j9kfk6656slq\" to finish deleting: GET https://rp.j3139200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/758cadb0-636e-4143-b1b5-5771c89ce22a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/758cadb0-636e-4143-b1b5-5771c89ce22a\",\n      \"name\": \"758cadb0-636e-4143-b1b5-5771c89ce22a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T02:09:52.194794521Z\",\n      \"endTime\": \"2026-08-20T02:33:57.777012859Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090277524570378240","occurred_at":"2026-08-20T03:19:31Z","signature_id":"07359dd26fae5073ad07b23e2966b25ea8b7f336630101293a6a95a60847a6b1","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090277524570378240","occurred_at":"2026-08-20T03:19:31Z","signature_id":"8d0ad4d2f288a6b561e87de5504ec0054fced4d300c4b16a0b8de371406bc244","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248456953139200","occurred_at":"2026-08-20T01:23:55Z","signature_id":"2650886f97daa2bac726adb3cee2f77ef1eb8b00325d923a67d6bb25b6830c20","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090222553485283328","occurred_at":"2026-08-19T23:41:05Z","signature_id":"22848fa078d0328825e14eed64b734b1953af1ba5f960122db077972a9bf624c","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090222553485283328","occurred_at":"2026-08-19T23:41:05Z","signature_id":"99ed3252f28c8c1c60792d87d919d200608c9ec9477676f42ad1850bb1179ba1","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090222553485283328","occurred_at":"2026-08-19T23:41:05Z","signature_id":"db7bf377a9e008ff66c454905ea366296951bc25fbb317aa85375f3caeab10d2","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090192872639827968","occurred_at":"2026-08-19T21:43:09Z","signature_id":"1d4cf42db36be45b0ad4ad46fd8aef7274ba52884b1241cf883d6084345a92a8","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090192872639827968","occurred_at":"2026-08-19T21:43:09Z","signature_id":"5527339989c73b7db0113c00563acb3179d3cd3ef9058e6068498b51fbc57c43","pr_number":6612,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090157821952790528","occurred_at":"2026-08-19T19:23:52Z","signature_id":"05c7f175202f00963e3302472b038a8f4574705534ee660adf9aa07ee10b0c98","pr_number":6612,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7fa5ceb8cdc856d9852c9106d45ffc1d35ac035ca27214fa5e56a7315735c71c","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: applydesires, nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":9,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.34071550255536626,"anchor_occurrences":9,"after_last_push_count":0,"trend_counts":[0,7,2,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-19T14:06:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001108438\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-gchzdq\" in resourcegroup=\"rg-np-skip-minor-gchzdq-ww2xhlmtfnwr\" to finish deleting: GET https://rp.j0627968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf1121db-2117-4096-a637-d4dd852c6573\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf1121db-2117-4096-a637-d4dd852c6573\",\n      \"name\": \"bf1121db-2117-4096-a637-d4dd852c6573\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:31:16.470227528Z\",\n      \"endTime\": \"2026-08-19T15:55:21.296940766Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9n9h5qdf8gb4n07q0k99jcbmc1n5ti still exists (deletion dispatched at 2026-08-19T15:31:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 12 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001108438\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-gchzdq\" in resourcegroup=\"rg-np-skip-minor-gchzdq-ww2xhlmtfnwr\" to finish deleting: GET https://rp.j0627968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf1121db-2117-4096-a637-d4dd852c6573\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf1121db-2117-4096-a637-d4dd852c6573\",\n      \"name\": \"bf1121db-2117-4096-a637-d4dd852c6573\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:31:16.470227528Z\",\n      \"endTime\": \"2026-08-19T15:55:21.296940766Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9n9h5qdf8gb4n07q0k99jcbmc1n5ti still exists (deletion dispatched at 2026-08-19T15:31:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 12 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012ab380\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-nf58gl6nvj9t\" to finish deleting: GET https://rp.j0627968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ceafc180-72fe-411b-afff-85006650127a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ceafc180-72fe-411b-afff-85006650127a\",\n      \"name\": \"ceafc180-72fe-411b-afff-85006650127a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:09:54.54926661Z\",\n      \"endTime\": \"2026-08-19T15:34:00.454237581Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9n9l7r4r26t0e2p0ms6h9a78nd8l01 still exists (deletion dispatched at 2026-08-19T15:09:54Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 12 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012ab380\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-nf58gl6nvj9t\" to finish deleting: GET https://rp.j0627968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ceafc180-72fe-411b-afff-85006650127a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ceafc180-72fe-411b-afff-85006650127a\",\n      \"name\": \"ceafc180-72fe-411b-afff-85006650127a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:09:54.54926661Z\",\n      \"endTime\": \"2026-08-19T15:34:00.454237581Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9n9l7r4r26t0e2p0ms6h9a78nd8l01 still exists (deletion dispatched at 2026-08-19T15:09:54Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 12 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010520a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-5862db\" in resourcegroup=\"rg-np-version-upgrade-5862db-r4qshvttzsb5\" to finish deleting: GET https://rp.j0479744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/778bb87c-ad19-4ddc-8bb3-8e955762f03f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/778bb87c-ad19-4ddc-8bb3-8e955762f03f\",\n      \"name\": \"778bb87c-ad19-4ddc-8bb3-8e955762f03f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T13:21:09.071042218Z\",\n      \"endTime\": \"2026-08-18T13:45:14.721211864Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s90f4mfatb3csd66rn5kibpmhvf12e1 still exists (deletion dispatched at 2026-08-18T13:21:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 12 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010520a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-5862db\" in resourcegroup=\"rg-np-version-upgrade-5862db-r4qshvttzsb5\" to finish deleting: GET https://rp.j0479744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/778bb87c-ad19-4ddc-8bb3-8e955762f03f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/778bb87c-ad19-4ddc-8bb3-8e955762f03f\",\n      \"name\": \"778bb87c-ad19-4ddc-8bb3-8e955762f03f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T13:21:09.071042218Z\",\n      \"endTime\": \"2026-08-18T13:45:14.721211864Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s90f4mfatb3csd66rn5kibpmhvf12e1 still exists (deletion dispatched at 2026-08-18T13:21:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 12 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"393e8fed94117bab564558b9b1b4cfb0c3f672fbd28d6d49c2152859b96335c3","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"59f960a2f1de30f6968c68d5db08823bb87f1a46ad72088af6549f8925fd368c","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"4b523dc95e31c68587ef2e29bb91e54f6ed234a2cbb116211ae5491d6a043fbc","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"5b67dda31fc986f569b5fe3acc6d8f15cd3560a26c56a593f54872a76df2e048","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"6f316a5aa51fa6e587e8149d83a3bf18ff90fa280ce95cc11e688e9cc51f5a0e","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"72bd4c6e8701bcc41097bda11c3a444a787b70471e2fc39ac1bea2e8dddb7cb7","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"a5477f87dc3a90b18e3ffcf717fde8a753c5cae7a5c308e1e255ef67635a9ce7","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"b74b27a3d560b33e1392cc2be54c24c4938d3c33209cf60b02d5260bcb4cd9c7","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"fbd955e36cb63a16d767dd642d07d5f636752f40c089a28994c87265de212aa1","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e405dc3ba5eaa76b269136ece3c7ec09bc9d5c543da581d8b5e14a4da0eb2410","failure_pattern":"ERROR CODE: DeploymentStackDeploymentFailed","search_query":"ERROR CODE: DeploymentStackDeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Monitoring/regional/monitoring","occurrences":8,"runs_affected":8,"failed_runs":8,"run_impact_percent":1.362862010221465,"anchor_occurrences":8,"after_last_push_count":0,"trend_counts":[0,2,1,3,2,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":10,"prior_last_seen_at":"2026-08-21T21:59:03Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Monitoring/regional/monitoring","occurrences":8}],"full_error_samples":["time=2026-08-21T22:06:45.798Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring description=\"Step monitoring\\n  Kind: ARMStack\\n  Template: templates/monitoring.bicep\\n  Parameters: configurations/monitoring.tmpl.bicepparam\"\ntime=2026-08-21T22:07:37.070Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring err=\"failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/37a71f39-392c-4236-b7bc-d450dfa4c5b6\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/37a71f39-392c-4236-b7bc-d450dfa4c5b6\\\",\\n  \\\"name\\\": \\\"37a71f39-392c-4236-b7bc-d450dfa4c5b6\\\",\\n  \\\"status\\\": \\\"failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n    \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: 'f47e133e-49d1-4df1-8690-f91ad3063ea3'.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4648448/providers/Microsoft.Resources/deployments/y5flko475afn22dntfyewzfj7gfkbiwdmj7ezjquufwse-260821221zuwz\\\",\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n        \\\"details\\\": [\\n          {\\n            \\\"code\\\": \\\"BadRequest\\\",\\n            \\\"message\\\": \\\"Property name: Rules[3].Expression, Attempted value: null, Error: 1:110: parse error: unknown escape sequence U+002E '.' Activity ID: 579c60d6-5bc9-4a52-b767-c8d018c5f149.\\\"\\n          }\\n        ]\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-08-21T21:50:06.716Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring description=\"Step monitoring\\n  Kind: ARMStack\\n  Template: templates/monitoring.bicep\\n  Parameters: configurations/monitoring.tmpl.bicepparam\"\ntime=2026-08-21T21:51:27.408Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring err=\"failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/dadf4cf0-aee7-46d3-92d2-f8fc4f1e4cab\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/dadf4cf0-aee7-46d3-92d2-f8fc4f1e4cab\\\",\\n  \\\"name\\\": \\\"dadf4cf0-aee7-46d3-92d2-f8fc4f1e4cab\\\",\\n  \\\"status\\\": \\\"failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n    \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: 'c8f9f91e-9264-4c83-804f-b9f2716a3570'.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8878848/providers/Microsoft.Resources/deployments/y5flko475afn22dntfyewzfj7gfkbiwdmj7ezjquufwse-26082121eod0o\\\",\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n        \\\"details\\\": [\\n          {\\n            \\\"code\\\": \\\"BadRequest\\\",\\n            \\\"message\\\": \\\"Property name: Rules[3].Expression, Attempted value: null, Error: 1:110: parse error: unknown escape sequence U+002E '.' Activity ID: 64eee31e-d466-4285-abd9-d9333b56d87c.\\\"\\n          }\\n        ]\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-08-20T03:05:18.803Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring description=\"Step monitoring\\n  Kind: ARMStack\\n  Template: templates/monitoring.bicep\\n  Parameters: configurations/monitoring.tmpl.bicepparam\"\ntime=2026-08-20T03:06:10.220Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Monitoring resourceGroup=regional step=monitoring err=\"failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/32d5ddb3-12f2-4f69-9adc-281d5a5aee90\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/32d5ddb3-12f2-4f69-9adc-281d5a5aee90\\\",\\n  \\\"name\\\": \\\"32d5ddb3-12f2-4f69-9adc-281d5a5aee90\\\",\\n  \\\"status\\\": \\\"failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n    \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: 'ed733f51-4922-4e52-8437-524f6699e679'.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3579264/providers/Microsoft.Resources/deployments/y5flko475afn22dntfyewzfj7gfkbiwdmj7ezjquufwse-260820031kmin\\\",\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n        \\\"details\\\": [\\n          {\\n            \\\"code\\\": \\\"BadRequest\\\",\\n            \\\"message\\\": \\\"Property name: Rules[3].Expression, Attempted value: null, Error: 1:110: parse error: unknown escape sequence U+002E '.' Activity ID: b6b6ddad-1c68-456d-966d-a061c184aaf2.\\\"\\n          }\\n        ]\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090921650764648448","occurred_at":"2026-08-21T21:59:03Z","signature_id":"667b7b6f874599a25c1c7daf7f0f9a64363882769e60c1224517a12577215162","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090912921788878848","occurred_at":"2026-08-21T21:24:22Z","signature_id":"667b7b6f874599a25c1c7daf7f0f9a64363882769e60c1224517a12577215162","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090266981373579264","occurred_at":"2026-08-20T02:37:38Z","signature_id":"667b7b6f874599a25c1c7daf7f0f9a64363882769e60c1224517a12577215162","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090256601465229312","occurred_at":"2026-08-20T01:56:23Z","signature_id":"667b7b6f874599a25c1c7daf7f0f9a64363882769e60c1224517a12577215162","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090233522550738944","occurred_at":"2026-08-20T00:24:41Z","signature_id":"667b7b6f874599a25c1c7daf7f0f9a64363882769e60c1224517a12577215162","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090165593054384128","occurred_at":"2026-08-19T19:54:45Z","signature_id":"667b7b6f874599a25c1c7daf7f0f9a64363882769e60c1224517a12577215162","pr_number":6614,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089852729714282496","occurred_at":"2026-08-18T23:11:32Z","signature_id":"667b7b6f874599a25c1c7daf7f0f9a64363882769e60c1224517a12577215162","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089843308665769984","occurred_at":"2026-08-18T22:34:06Z","signature_id":"667b7b6f874599a25c1c7daf7f0f9a64363882769e60c1224517a12577215162","pr_number":6554,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bad17b00fd2861c199fb8c7332ad5eb58ff1eb35d64efc7066c892df3da78afa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] \u003cno_message\u003e; [clusterServiceStatus] \u003cno_message\u003e; [descendantResources] \u003cno_message\u003e; [hostedCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":1.192504258943782,"anchor_occurrences":7,"after_last_push_count":7,"also_in":["STG"],"trend_counts":[1,1,1,2,2,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":19,"prior_last_seen_at":"2026-08-21T04:29:50Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c982a0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-gdnhnw\" in resourcegroup=\"rg-hs-pre--hm7prkdxbkw6\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67ed1acc-0077-4b07-848d-484f46170f44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67ed1acc-0077-4b07-848d-484f46170f44\",\n      \"name\": \"67ed1acc-0077-4b07-848d-484f46170f44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:21:50.42303027Z\",\n      \"endTime\": \"2026-08-24T15:45:59.872804145Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c982a0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-gdnhnw\" in resourcegroup=\"rg-hs-pre--hm7prkdxbkw6\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67ed1acc-0077-4b07-848d-484f46170f44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67ed1acc-0077-4b07-848d-484f46170f44\",\n      \"name\": \"67ed1acc-0077-4b07-848d-484f46170f44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:21:50.42303027Z\",\n      \"endTime\": \"2026-08-24T15:45:59.872804145Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001fe00d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-sbns7nl57fsx\" to finish deleting: GET https://rp.j2704896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02c893ba-d638-4b2d-9ac6-b24d9ffc2c41\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02c893ba-d638-4b2d-9ac6-b24d9ffc2c41\",\n      \"name\": \"02c893ba-d638-4b2d-9ac6-b24d9ffc2c41\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T05:44:31.451258644Z\",\n      \"endTime\": \"2026-08-21T06:08:32.645558792Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001fe00d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-sbns7nl57fsx\" to finish deleting: GET https://rp.j2704896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02c893ba-d638-4b2d-9ac6-b24d9ffc2c41\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/02c893ba-d638-4b2d-9ac6-b24d9ffc2c41\",\n      \"name\": \"02c893ba-d638-4b2d-9ac6-b24d9ffc2c41\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T05:44:31.451258644Z\",\n      \"endTime\": \"2026-08-21T06:08:32.645558792Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001524090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-znkgn5hxfbf9\" to finish deleting: GET https://rp.j3708160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c4fec91-844b-4c62-aa4f-0c1bd98999c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c4fec91-844b-4c62-aa4f-0c1bd98999c6\",\n      \"name\": \"7c4fec91-844b-4c62-aa4f-0c1bd98999c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:32:47.45529919Z\",\n      \"endTime\": \"2026-08-21T02:56:48.748022108Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001524090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-znkgn5hxfbf9\" to finish deleting: GET https://rp.j3708160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c4fec91-844b-4c62-aa4f-0c1bd98999c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c4fec91-844b-4c62-aa4f-0c1bd98999c6\",\n      \"name\": \"7c4fec91-844b-4c62-aa4f-0c1bd98999c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:32:47.45529919Z\",\n      \"endTime\": \"2026-08-21T02:56:48.748022108Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","signature_id":"cf9ddb524cae2c0c6a37a93000fda2cd14fe56bff50daec6b5b3ed77ad21467c","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"bee1e84ff0a64e4ca2e0552272b055ce62b94e39bd608e5a2e2955aefb0ca0e9","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090612430223708160","occurred_at":"2026-08-21T01:30:11Z","signature_id":"bd964ef4ccdbb7f347258f9538766dbd5e26f438ced91675a269619c3c825f06","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6621/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514755771961344","occurred_at":"2026-08-20T19:02:12Z","signature_id":"c15560f309da4b6beb1a22ff7a9cc58001804d18d46863bfe0ac76d9d56dac64","pr_number":6621,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090497381098328064","occurred_at":"2026-08-20T17:53:09Z","signature_id":"e618604f8ab9e6904abacd4ab44cb3e1e0af728093b8df4f6cf4ffee68b2721f","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090111264041734144","occurred_at":"2026-08-19T16:18:52Z","signature_id":"c6d70da266bb361671887556071e72f8a400a1404bcfa0d0aff64c5103e4e217","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089759198756737024","occurred_at":"2026-08-18T16:59:53Z","signature_id":"c293dc468dfcc11091673e64791bf0596f09f5af8b19f3f592a336acbd663a15","pr_number":6588,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b952bd87cc064f775e96c64b34d1d595213c1b657abf569b98d2c49605644533","failure_pattern":"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4","search_query":"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":1.192504258943782,"anchor_occurrences":7,"after_last_push_count":6,"trend_counts":[2,4,1,1,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":19,"prior_last_seen_at":"2026-08-21T21:20:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":6}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:194]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0011f9ad0\u003e: \n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21, targetMinor=4.22) failed: clusterversion status.history has no version in target minor \"4.22\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:194]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0011f9ad0\u003e: \n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21, targetMinor=4.22) failed: clusterversion status.history has no version in target minor \"4.22\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.005s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f8a348\u003e: \n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21, targetMinor=4.22) failed: clusterversion status.history has no version in target minor \"4.22\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.005s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f8a348\u003e: \n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21, targetMinor=4.22) failed: clusterversion status.history has no version in target minor \"4.22\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.005s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001a2ab10\u003e: \n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21, targetMinor=4.22) failed: clusterversion status.history has no version in target minor \"4.22\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.005s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001a2ab10\u003e: \n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21, targetMinor=4.22) failed: clusterversion status.history has no version in target minor \"4.22\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090912019128520704","occurred_at":"2026-08-21T21:20:47Z","signature_id":"105cc56e52a2d552e4ddbe874bd058b495386714220d4a05879a34d8129b2709","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"7b88e73ef704bd58321b40735ba98a518a526cd77c56eddfbc7f78f59d7dfb88","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090079394612645888","occurred_at":"2026-08-19T14:12:08Z","signature_id":"e4ce4e490406651d21d2d7a2c92ef7cf75c01173be981a5167bf0f960c11f06a","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089829163719462912","occurred_at":"2026-08-18T21:37:54Z","signature_id":"054954f52f101f466477799d820b277d696047522101d2d7392dce97a053ddca","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089723779814002688","occurred_at":"2026-08-18T14:39:04Z","signature_id":"a58cd665be2669caa01a6477eb01fff3897c4f23cbc283781e3f645ea63e4b20","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708056525934592","occurred_at":"2026-08-18T13:36:10Z","signature_id":"6d71a558948d648292c08c6e16464dd0ca50827dfd2f41f3de243939f195e45e","pr_number":6585,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"0a4171a2ed68da4a01dbf9f7b3e89d4c853a9fdbc77e15f7e988b7eca3d8ea92","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":1.192504258943782,"anchor_occurrences":7,"after_last_push_count":9,"trend_counts":[2,1,3,0,1,1,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":28,"prior_last_seen_at":"2026-08-22T04:44:09Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":7}],"full_error_samples":["alert fired 6 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T18:15:41Z\n  Ended:       2026-08-24T18:21:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T18:15:41Z\n  Ended:       2026-08-24T18:22:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-08-24T18:15:41Z\n  Ended:       2026-08-24T18:22:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-08-24T18:15:41Z\n  Ended:       2026-08-24T18:23:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-08-24T18:15:41Z\n  Ended:       2026-08-24T18:22:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-08-24T18:15:41Z\n  Ended:       2026-08-24T18:23:41Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0375808-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.140:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0375808/providers/microsoft.monitor/accounts/services-j0375808\", namespace=\"klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k\", pod=\"arohcp-monitor-kube-state-metrics-5b9fdd8b5b-hf5qz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sd37mkptl2qprba4ejkvlbldgl77u9k/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","alert fired 6 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-22T06:11:54Z\n  Ended:       2026-08-22T06:23:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/services-j8029824\", namespace=\"klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-22T06:11:54Z\n  Ended:       2026-08-22T06:17:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/services-j8029824\", namespace=\"klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-08-22T06:11:54Z\n  Ended:       2026-08-22T06:21:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/services-j8029824\", namespace=\"klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-08-22T06:11:54Z\n  Ended:       2026-08-22T06:23:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/services-j8029824\", namespace=\"klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-08-22T06:11:54Z\n  Ended:       2026-08-22T06:19:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/services-j8029824\", namespace=\"klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-08-22T06:11:54Z\n  Ended:       2026-08-22T06:19:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8029824-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.144:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8029824/providers/microsoft.monitor/accounts/services-j8029824\", namespace=\"klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-f54dn\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2sbeee9sknhjmru28j861qqr85i8o22i/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T03:07:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6439552-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.130:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6439552/providers/microsoft.monitor/accounts/services-j6439552\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-6t89j\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T03:33:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6439552-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.130:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6439552/providers/microsoft.monitor/accounts/services-j6439552\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-6t89j\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091931812560375808","occurred_at":"2026-08-24T16:43:08Z","signature_id":"a6a83541f08b4359551ade88cceb74431bde5d979f13cb6852b8c8bc1013fd98","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091023619198029824","occurred_at":"2026-08-22T04:44:09Z","signature_id":"317b8c5665259b143d908887f090198d5122fed07b3f6eabfcb499b51735af2d","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"263f0fec58eb18ce8eb5b06529e96eb0994337f8a7d3cdff35b43d545a99a4cd","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090148755918557184","occurred_at":"2026-08-19T18:47:51Z","signature_id":"57fd6a38bf423a3d6bdded79b3ceae9e1993465036803049a90f5483d3e998d2","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"a403c540378f317853254227103a4fcdfcefbdd1200a6e0613ee5843ecd53feb","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089990952641892352","occurred_at":"2026-08-19T08:20:47Z","signature_id":"dbdb72f4c8d3baffa07ee83281d500e2b52443ca8d4794bedc1eed79717204c6","pr_number":6538,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725920976834560","occurred_at":"2026-08-18T14:47:39Z","signature_id":"55094ac7750143d15f2d8ef867b1debe5c8d46e71becc86c7f585af46507d596","pr_number":6527,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1c7abf95f6b325e2b7136a3af3c492c951894fb6cb7083b18c5b7aad3165e0e0","failure_pattern":"alert [hcp] KubeContainerWaiting fired","search_query":"alert [hcp] KubeContainerWaiting fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeContainerWaiting does not fire","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":1.0221465076660987,"anchor_occurrences":6,"after_last_push_count":8,"trend_counts":[4,4,2,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":40,"prior_last_seen_at":"2026-08-19T18:19:42Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeContainerWaiting does not fire","occurrences":6}],"full_error_samples":["alert fired 8 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T20:14:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1797504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9r1sjk7nl7cq2gcvnm7d10nbm2ae5f-i2j7w2x8e1s9q5g\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9r1sjk7nl7cq2gcvnm7d10nbm2ae5f-i2j7w2x8e1s9q5g on container healthz has been in waiting state for longer than 1 hour.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T20:14:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1797504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9r1sjk7nl7cq2gcvnm7d10nbm2ae5f-i2j7w2x8e1s9q5g\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9r1sjk7nl7cq2gcvnm7d10nbm2ae5f-i2j7w2x8e1s9q5g on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T20:14:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1797504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9r1sjk7nl7cq2gcvnm7d10nbm2ae5f-i2j7w2x8e1s9q5g\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9r1sjk7nl7cq2gcvnm7d10nbm2ae5f-i2j7w2x8e1s9q5g on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T20:14:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1797504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9r1sjk7nl7cq2gcvnm7d10nbm2ae5f-i2j7w2x8e1s9q5g\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9r1sjk7nl7cq2gcvnm7d10nbm2ae5f-i2j7w2x8e1s9q5g on container etcd has been in waiting state for longer than 1 hour.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T20:15:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1797504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9r2ec2tegi43jr5a45h7ojphgg4rhv-f0v3l6y2u5x0q0u\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9r2ec2tegi43jr5a45h7ojphgg4rhv-f0v3l6y2u5x0q0u on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T20:15:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1797504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9r2ec2tegi43jr5a45h7ojphgg4rhv-f0v3l6y2u5x0q0u\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9r2ec2tegi43jr5a45h7ojphgg4rhv-f0v3l6y2u5x0q0u on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T20:15:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1797504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9r2ec2tegi43jr5a45h7ojphgg4rhv-f0v3l6y2u5x0q0u\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9r2ec2tegi43jr5a45h7ojphgg4rhv-f0v3l6y2u5x0q0u on container etcd has been in waiting state for longer than 1 hour.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T20:15:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j1797504-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9r2ec2tegi43jr5a45h7ojphgg4rhv-f0v3l6y2u5x0q0u\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9r2ec2tegi43jr5a45h7ojphgg4rhv-f0v3l6y2u5x0q0u on container healthz has been in waiting state for longer than 1 hour.","alert fired 45 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:55:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"manager\", namespace=\"ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q\", pod=\"capi-provider-58999f64dc-bltc4\", severity=\"warning\"\n  Description: pod/capi-provider-58999f64dc-bltc4 in namespace ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q on container manager has been in waiting state for longer than 1 hour.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:56:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h on container etcd has been in waiting state for longer than 1 hour.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:56:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:56:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q\", pod=\"etcd-0\", severity=\"warning\"\n  Description: pod/etcd-0 in namespace ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q on container etcd has been in waiting state for longer than 1 hour.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:56:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:56:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q\", pod=\"etcd-0\", severity=\"warning\"\n  Description: pod/etcd-0 in namespace ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:56:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q\", pod=\"etcd-0\", severity=\"warning\"\n  Description: pod/etcd-0 in namespace ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q on container healthz has been in waiting state for longer than 1 hour.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:56:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q\", pod=\"etcd-0\", severity=\"warning\"\n  Description: pod/etcd-0 in namespace ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:56:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h on container healthz has been in waiting state for longer than 1 hour.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e on container healthz has been in waiting state for longer than 1 hour.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f on container etcd has been in waiting state for longer than 1 hour.\nFiring 13:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster on container etcd has been in waiting state for longer than 1 hour.\nFiring 14:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 15:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e on container etcd has been in waiting state for longer than 1 hour.\nFiring 16:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f on container healthz has been in waiting state for longer than 1 hour.\nFiring 17:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 18:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster on container healthz has been in waiting state for longer than 1 hour.\nFiring 19:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s on container healthz has been in waiting state for longer than 1 hour.\nFiring 20:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 21:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s on container etcd has been in waiting state for longer than 1 hour.\nFiring 22:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 23:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 24:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 25:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:57:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 26:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:58:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 27:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:58:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d on container etcd has been in waiting state for longer than 1 hour.\nFiring 28:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:58:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d on container healthz has been in waiting state for longer than 1 hour.\nFiring 29:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:58:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r on container etcd has been in waiting state for longer than 1 hour.\nFiring 30:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:58:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 31:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:58:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r on container healthz has been in waiting state for longer than 1 hour.\nFiring 32:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:58:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 33:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:58:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 34:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:59:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress on container healthz has been in waiting state for longer than 1 hour.\nFiring 35:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:59:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress on container etcd has been in waiting state for longer than 1 hour.\nFiring 36:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:59:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 37:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:59:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 38:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:01:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 39:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:01:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 40:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:01:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster on container healthz has been in waiting state for longer than 1 hour.\nFiring 41:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:01:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 42:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:01:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster on container etcd has been in waiting state for longer than 1 hour.\nFiring 43:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:01:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster on container etcd has been in waiting state for longer than 1 hour.\nFiring 44:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:01:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 45:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:01:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster\", pod=\"etcd-2\", severity=\"warning\"\n  Description: pod/etcd-2 in namespace ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster on container healthz has been in waiting state for longer than 1 hour.","alert fired 8 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-18T23:21:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j9462912-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s98mmkm7hr956a64f4ud6162fd7al26-f4m8c9s8m7i5v9t\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s98mmkm7hr956a64f4ud6162fd7al26-f4m8c9s8m7i5v9t on container etcd has been in waiting state for longer than 1 hour.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-18T23:21:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j9462912-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s98mmkm7hr956a64f4ud6162fd7al26-f4m8c9s8m7i5v9t\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s98mmkm7hr956a64f4ud6162fd7al26-f4m8c9s8m7i5v9t on container etcd-metrics has been in waiting state for longer than 1 hour.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-18T23:21:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j9462912-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s98mmkm7hr956a64f4ud6162fd7al26-f4m8c9s8m7i5v9t\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s98mmkm7hr956a64f4ud6162fd7al26-f4m8c9s8m7i5v9t on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-18T23:21:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j9462912-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s98mmkm7hr956a64f4ud6162fd7al26-f4m8c9s8m7i5v9t\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s98mmkm7hr956a64f4ud6162fd7al26-f4m8c9s8m7i5v9t on container healthz has been in waiting state for longer than 1 hour.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-08-18T23:22:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j9462912-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd-defrag\", namespace=\"ocm-arohcpci01-2s98mlbgmtvccvr8mthq6lnumiqur3oo-s6y8z3g8p8j2h8f\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s98mlbgmtvccvr8mthq6lnumiqur3oo-s6y8z3g8p8j2h8f on container etcd-defrag has been in waiting state for longer than 1 hour.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-08-18T23:22:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j9462912-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", namespace=\"ocm-arohcpci01-2s98mlbgmtvccvr8mthq6lnumiqur3oo-s6y8z3g8p8j2h8f\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s98mlbgmtvccvr8mthq6lnumiqur3oo-s6y8z3g8p8j2h8f on container etcd has been in waiting state for longer than 1 hour.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-08-18T23:22:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j9462912-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"healthz\", namespace=\"ocm-arohcpci01-2s98mlbgmtvccvr8mthq6lnumiqur3oo-s6y8z3g8p8j2h8f\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s98mlbgmtvccvr8mthq6lnumiqur3oo-s6y8z3g8p8j2h8f on container healthz has been in waiting state for longer than 1 hour.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-08-18T23:22:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j9462912-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd-metrics\", namespace=\"ocm-arohcpci01-2s98mlbgmtvccvr8mthq6lnumiqur3oo-s6y8z3g8p8j2h8f\", pod=\"etcd-1\", severity=\"warning\"\n  Description: pod/etcd-1 in namespace ocm-arohcpci01-2s98mlbgmtvccvr8mthq6lnumiqur3oo-s6y8z3g8p8j2h8f on container etcd-metrics has been in waiting state for longer than 1 hour."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"015cf07ce6f663064e0b25d73a7d2265851a63a80888900249935eba2609bfae","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"638796d67faa614987f4d56ac4f93565ff06ae312263ef561431a0eaf8bad2d8","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089829163719462912","occurred_at":"2026-08-18T21:37:54Z","signature_id":"2e8c5811c12b9128bae6a9f52954f5e62cd9d0060d073f702781d901b29baed7","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089713360550498304","occurred_at":"2026-08-18T13:57:44Z","signature_id":"8e26ecf8bf2596999a5bdd1b7ab62472d6ea35221d44e217aa6856d98b7239ff","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708056525934592","occurred_at":"2026-08-18T13:36:10Z","signature_id":"ab13b5ec9cb218a27992399c2f740659062832b1cf84cae075bb1df44abba832","pr_number":6585,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"df0e6747c9eb3ce3adc5b9e78666f5d5a69f715de0df5b23c2adbab4ba7db09c","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fac505f52469073fa6a64250c64e44813d69018a355acebf1fdde81a505c53f0","failure_pattern":"expected 3 nodes to have original label key1=value1","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:259]: expected 3 nodes to have original label key1=value1","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":1.0221465076660987,"anchor_occurrences":6,"after_last_push_count":0,"trend_counts":[0,2,1,0,2,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-10","2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-21T15:05:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":6}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:259]: expected 3 nodes to have original label key1=value1\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:259]: expected 3 nodes to have original label key1=value1\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091855258673221632","occurred_at":"2026-08-24T11:48:53Z","signature_id":"c01b4fcf6cc6dec59f03daaf85e15cd790cfdad06e2936a348917bd71e8e78e7","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090817534927310848","occurred_at":"2026-08-21T15:05:19Z","signature_id":"c01b4fcf6cc6dec59f03daaf85e15cd790cfdad06e2936a348917bd71e8e78e7","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090766839033368576","occurred_at":"2026-08-21T11:43:53Z","signature_id":"c01b4fcf6cc6dec59f03daaf85e15cd790cfdad06e2936a348917bd71e8e78e7","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"c01b4fcf6cc6dec59f03daaf85e15cd790cfdad06e2936a348917bd71e8e78e7","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"c01b4fcf6cc6dec59f03daaf85e15cd790cfdad06e2936a348917bd71e8e78e7","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089644114403397632","occurred_at":"2026-08-18T09:22:35Z","signature_id":"c01b4fcf6cc6dec59f03daaf85e15cd790cfdad06e2936a348917bd71e8e78e7","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.8517887563884157,"anchor_occurrences":5,"after_last_push_count":2,"also_in":["PROD"],"trend_counts":[0,1,1,1,0,1,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":10,"prior_last_seen_at":"2026-08-22T00:26:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010deff0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-4cmmkc\" in resourcegroup=\"admin-credential-lifecycle-test-mk5bt6ghv29k\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0644f278-df05-4603-a10d-3a66a1f851d7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0644f278-df05-4603-a10d-3a66a1f851d7\",\n      \"name\": \"0644f278-df05-4603-a10d-3a66a1f851d7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:17:03.672410989Z\",\n      \"endTime\": \"2026-08-24T15:41:10.046088761Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tjfmep0gog9fd4pfkr5bnpc0n642 still exists (deletion dispatched at 2026-08-24T15:17:04Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010deff0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-4cmmkc\" in resourcegroup=\"admin-credential-lifecycle-test-mk5bt6ghv29k\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0644f278-df05-4603-a10d-3a66a1f851d7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0644f278-df05-4603-a10d-3a66a1f851d7\",\n      \"name\": \"0644f278-df05-4603-a10d-3a66a1f851d7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:17:03.672410989Z\",\n      \"endTime\": \"2026-08-24T15:41:10.046088761Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tjfmep0gog9fd4pfkr5bnpc0n642 still exists (deletion dispatched at 2026-08-24T15:17:04Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012ab8d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-khp84d8mdsh7\" to finish deleting: GET https://rp.j7976064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b538a56-05be-48db-bb47-4c5a33c4accf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b538a56-05be-48db-bb47-4c5a33c4accf\",\n      \"name\": \"7b538a56-05be-48db-bb47-4c5a33c4accf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:26:12.085661598Z\",\n      \"endTime\": \"2026-08-22T01:50:12.281269064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sbakg7dgt5f0fajefaqb6utb0vjshqh still exists (deletion dispatched at 2026-08-22T01:26:12Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012ab8d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-khp84d8mdsh7\" to finish deleting: GET https://rp.j7976064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b538a56-05be-48db-bb47-4c5a33c4accf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b538a56-05be-48db-bb47-4c5a33c4accf\",\n      \"name\": \"7b538a56-05be-48db-bb47-4c5a33c4accf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:26:12.085661598Z\",\n      \"endTime\": \"2026-08-22T01:50:12.281269064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sbakg7dgt5f0fajefaqb6utb0vjshqh still exists (deletion dispatched at 2026-08-22T01:26:12Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010d41e0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-tr4whf2bpfsf\" to finish deleting: GET https://rp.j3372032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e72e998-9d37-429a-9354-c729235c1e6f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e72e998-9d37-429a-9354-c729235c1e6f\",\n      \"name\": \"4e72e998-9d37-429a-9354-c729235c1e6f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T02:15:23.413243123Z\",\n      \"endTime\": \"2026-08-20T02:39:29.402250383Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa13enfvdgm47tbcg2mg1bln7e4ikq7 still exists (deletion dispatched at 2026-08-20T02:15:25Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0010d41e0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-tr4whf2bpfsf\" to finish deleting: GET https://rp.j3372032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e72e998-9d37-429a-9354-c729235c1e6f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4e72e998-9d37-429a-9354-c729235c1e6f\",\n      \"name\": \"4e72e998-9d37-429a-9354-c729235c1e6f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T02:15:23.413243123Z\",\n      \"endTime\": \"2026-08-20T02:39:29.402250383Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa13enfvdgm47tbcg2mg1bln7e4ikq7 still exists (deletion dispatched at 2026-08-20T02:15:25Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","signature_id":"7df46629584e4c4253c91624b483f13865f970598c9ed547ff855ac9eea68fe2","pr_number":6232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090958673797976064","occurred_at":"2026-08-22T00:26:10Z","signature_id":"b11845b41dfb98943e4686416824e8a527b84d128d07df3276550aa8a60f48ab","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090249792243372032","occurred_at":"2026-08-20T01:29:20Z","signature_id":"413ec903f1148685f58b502ff35e13104028e2eb95ffff3c388e1f10df7b1622","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090087196852752384","occurred_at":"2026-08-19T14:43:14Z","signature_id":"2d06bb34613aea8950ff84401b1d039dd84dbc4e5c72eb3633c6ca2db250b20b","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089796885723222016","occurred_at":"2026-08-18T19:29:38Z","signature_id":"d5570221f7c1485dfc090b198466ce5e8bcc557eeb14055f61b40ab1c21a8a11","pr_number":6592,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aedc4b61db3f58705ccf159035a4315019ba43dddaf5edf363aef4d927730152","failure_pattern":"Engineering should be able to retrieve expected metrics from the /metrics endpoint: Timed out after \u003cduration\u003es.","search_query":"Timed out after 1800.001s.","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve expected metrics from the /metrics endpoint","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.8517887563884157,"anchor_occurrences":5,"after_last_push_count":0,"trend_counts":[1,5,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-18T17:40:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve expected metrics from the /metrics endpoint","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:131]: Timed out after 1800.001s.\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:131]: Timed out after 1800.001s.\nExpected\n    \u003cbool\u003e: ...\nto be true","fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:131]: Timed out after 1800.000s.\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:131]: Timed out after 1800.000s.\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6587/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089769452286513152","occurred_at":"2026-08-18T17:40:38Z","signature_id":"83eb74118ab075e53463844b414c52c86808fa5f9d778a403f7ce8aaa25962c7","pr_number":6587,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089665224855523328","occurred_at":"2026-08-18T10:46:28Z","signature_id":"83eb74118ab075e53463844b414c52c86808fa5f9d778a403f7ce8aaa25962c7","pr_number":6573,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089615938193199104","occurred_at":"2026-08-18T07:30:25Z","signature_id":"83eb74118ab075e53463844b414c52c86808fa5f9d778a403f7ce8aaa25962c7","pr_number":6573,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089574646696382464","occurred_at":"2026-08-18T04:46:32Z","signature_id":"83eb74118ab075e53463844b414c52c86808fa5f9d778a403f7ce8aaa25962c7","pr_number":6573,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089512043190161408","occurred_at":"2026-08-18T00:37:46Z","signature_id":"8414f8df95d0521e983e3af649840de9b144c4ebe17a99d7effe8b6ab751ef8d","pr_number":6573,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"95c96916264bf9de45b37005d736b6d9552404339f828cf7083033bb2290c13f","failure_pattern":"alert [svc] FleetControllerRetryHotLoop fired","search_query":"alert [svc] FleetControllerRetryHotLoop fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FleetControllerRetryHotLoop does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.8517887563884157,"anchor_occurrences":5,"after_last_push_count":2,"trend_counts":[0,3,1,0,0,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-19T22:05:11Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FleetControllerRetryHotLoop does not fire","occurrences":5}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T17:38:07Z\n  Ended:       2026-08-24T17:45:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"FleetControllerRetryHotLoop\", cluster=\"ci01-j0629888-svc\", component=\"fleet\", name=\"clustersserviceregistrationcontroller\", severity=\"warning\"\n  Description: Fleet controller workqueue clustersserviceregistrationcontroller has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-19T22:56:27Z\n  Ended:       2026-08-19T23:03:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"FleetControllerRetryHotLoop\", cluster=\"ci01-j2195200-svc\", component=\"fleet\", name=\"clustersserviceregistrationcontroller\", severity=\"warning\"\n  Description: Fleet controller workqueue clustersserviceregistrationcontroller has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-18T17:59:28Z\n  Ended:       2026-08-18T18:05:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"FleetControllerRetryHotLoop\", cluster=\"ci01-j0818176-svc\", component=\"fleet\", name=\"clustersserviceregistrationcontroller\", severity=\"warning\"\n  Description: Fleet controller workqueue clustersserviceregistrationcontroller has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928685790629888","occurred_at":"2026-08-24T16:40:39Z","signature_id":"3eebb3220fdafc37e59ae5d05cda26d2e052d0d1681fedb42f8fa69446913217","pr_number":6591,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090198433062195200","occurred_at":"2026-08-19T22:05:11Z","signature_id":"3eebb3220fdafc37e59ae5d05cda26d2e052d0d1681fedb42f8fa69446913217","pr_number":6492,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089757851600818176","occurred_at":"2026-08-18T16:54:32Z","signature_id":"3eebb3220fdafc37e59ae5d05cda26d2e052d0d1681fedb42f8fa69446913217","pr_number":5880,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089722221688786944","occurred_at":"2026-08-18T14:32:57Z","signature_id":"3eebb3220fdafc37e59ae5d05cda26d2e052d0d1681fedb42f8fa69446913217","pr_number":6527,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089679732839813120","occurred_at":"2026-08-18T11:44:07Z","signature_id":"6f02029b5107c8c793b78573b60e1ea286bc1c30d67275dc1bb3af4002da8ea5","pr_number":6553,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"406d47e638f407d24104655febf18e753de33546f725682f49558d9bc4ae2365","failure_pattern":"private ingress should not be reachable from outside the VNet, but connection succeeded","search_query":"private ingress should not be reachable from outside the VNet, but connection succeeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":0.8517887563884157,"anchor_occurrences":5,"after_last_push_count":7,"trend_counts":[0,1,1,1,2,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":13,"prior_last_seen_at":"2026-08-21T21:06:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:186]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred.  Got:\n    \u003cnil\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:186]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred.  Got:\n    \u003cnil\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090908436404375552","occurred_at":"2026-08-21T21:06:33Z","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090618825308377088","occurred_at":"2026-08-21T01:55:44Z","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514791213830144","occurred_at":"2026-08-20T19:02:20Z","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090126361980047360","occurred_at":"2026-08-19T17:18:51Z","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6564/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725877364461568","occurred_at":"2026-08-18T14:47:25Z","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","pr_number":6564,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-bgsx4.apps.aro.b3q2k1n6p8k6q6p.5htp.j7087360.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-bgsx4.apps.aro.b3q2k1n6p8k6q6p.5htp.j7087360.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bgsx4.apps.aro.b3q2k1n6p8k6q6p.5htp.j7087360.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.6814310051107325,"anchor_occurrences":4,"after_last_push_count":3,"trend_counts":[0,1,1,0,0,0,0,2],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-19T09:11:12Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-fn4s2m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dea2e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-fn4s2m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dea2e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jhj46.apps.aro.o4q2a9i7j5z2k6t.i9i4.j4428544.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:131]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005da4a0\u003e: \n    DNS for route host agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:131]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005da4a0\u003e: \n    DNS for route host agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-gr4x8.apps.aro.v20260901.3e97.j6642304.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-rp2lrf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009b70e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-r92l5.apps.aro.h0k3z1f6y5f5g2r.rlny.j0859904.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-r92l5.apps.aro.h0k3z1f6y5f5g2r.rlny.j0859904.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-r92l5.apps.aro.h0k3z1f6y5f5g2r.rlny.j0859904.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-rp2lrf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009b70e0\u003e: \n    DNS for route host agnhost-e2e-sample-app-r92l5.apps.aro.h0k3z1f6y5f5g2r.rlny.j0859904.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-r92l5.apps.aro.h0k3z1f6y5f5g2r.rlny.j0859904.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-r92l5.apps.aro.h0k3z1f6y5f5g2r.rlny.j0859904.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091977495384428544","occurred_at":"2026-08-24T19:54:36Z","signature_id":"4a86fd68539d0c65cc2c794f0a3b117d62969a34ae1b535d87cc0cce6d88b528","pr_number":6629,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6618/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091811921836642304","occurred_at":"2026-08-24T08:56:40Z","signature_id":"67e0d9c3e541426a700c2b1097cab5407e75837246f0b2492c0fa4c512883fb9","pr_number":6618,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090003636900859904","occurred_at":"2026-08-19T09:11:12Z","signature_id":"9094f8b2d47d1b3fb610e6e3fed35fc3383395672932f3de5845d46d2dbfa706","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6426/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089706907987087360","occurred_at":"2026-08-18T13:32:06Z","signature_id":"df9630d12fe79a9290c67d45fe073bcf92fefc842e7b494a5c5254f58e009d62","pr_number":6426,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"293006e7a1ad208844fa10f4e2b3c0c8959662e799277a9c1d23e6534078c9e0","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-8fh9cjmrkw2p\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-8fh9cjmrkw2p\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-8fh9cjmrkw2p, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-8fh9cjmrkw2p\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-8fh9cjmrkw2p, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.6814310051107325,"anchor_occurrences":4,"after_last_push_count":2,"also_in":["PROD"],"trend_counts":[0,3,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":108,"prior_last_seen_at":"2026-08-19T14:35:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001590300\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-wcx6jgxkn8nc\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-wcx6jgxkn8nc\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-wcx6jgxkn8nc, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-wcx6jgxkn8nc\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-wcx6jgxkn8nc, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001590300\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-wcx6jgxkn8nc\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-wcx6jgxkn8nc\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-wcx6jgxkn8nc, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-wcx6jgxkn8nc\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-wcx6jgxkn8nc, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00283a3a8\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-8fh9cjmrkw2p\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-8fh9cjmrkw2p\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-8fh9cjmrkw2p, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-8fh9cjmrkw2p\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-8fh9cjmrkw2p, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00283a3a8\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-8fh9cjmrkw2p\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-8fh9cjmrkw2p\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-8fh9cjmrkw2p, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-8fh9cjmrkw2p\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-8fh9cjmrkw2p, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001460120\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-version-upgrade-2gqc2c-r66f7w5n66xd\", caused by: failed waiting for hcpCluster=\"np-version-upgrade-cluster-2gqc2c\" in resourcegroup=\"rg-np-version-upgrade-2gqc2c-r66f7w5n66xd\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-version-upgrade-cluster-2gqc2c in resource group rg-np-version-upgrade-2gqc2c-r66f7w5n66xd, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-version-upgrade-cluster-2gqc2c\" in resourcegroup=\"rg-np-version-upgrade-2gqc2c-r66f7w5n66xd\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-version-upgrade-cluster-2gqc2c in resource group rg-np-version-upgrade-2gqc2c-r66f7w5n66xd, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001460120\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-version-upgrade-2gqc2c-r66f7w5n66xd\", caused by: failed waiting for hcpCluster=\"np-version-upgrade-cluster-2gqc2c\" in resourcegroup=\"rg-np-version-upgrade-2gqc2c-r66f7w5n66xd\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-version-upgrade-cluster-2gqc2c in resource group rg-np-version-upgrade-2gqc2c-r66f7w5n66xd, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-version-upgrade-cluster-2gqc2c\" in resourcegroup=\"rg-np-version-upgrade-2gqc2c-r66f7w5n66xd\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-version-upgrade-cluster-2gqc2c in resource group rg-np-version-upgrade-2gqc2c-r66f7w5n66xd, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090085264507539456","occurred_at":"2026-08-19T14:35:33Z","signature_id":"da5ef245b7e16db299546afb081bf622baf8f64d77b67033bc19a14385ca9f5d","pr_number":6607,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089809279165927424","occurred_at":"2026-08-18T20:18:53Z","signature_id":"4e799115adec101ebed8130ab793d035e622c7c9d899efa5f1832e70da30fc7d","pr_number":6562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089796885723222016","occurred_at":"2026-08-18T19:29:38Z","signature_id":"0122057675b8a41813124e40d1740f62abf5311baa6df5141bc4342de9040e9a","pr_number":6592,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089651610870353920","occurred_at":"2026-08-18T09:52:22Z","signature_id":"b32a9b1f62c3b861cfdae56d10c072e1da6db3a7d0ae455614a6db9e33f58b06","pr_number":6562,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"03191c7ee6ef10f05815abde7f8b6f6bad2b4b3b1acdea0c834f5ddce823bc91","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: externalAuths, nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.6814310051107325,"anchor_occurrences":4,"after_last_push_count":3,"also_in":["STG"],"trend_counts":[1,2,1,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":7,"prior_last_seen_at":"2026-08-20T23:23:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012201b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-fnngq6qtshfw\" to finish deleting: GET https://rp.j8029568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26314739-180a-45be-9bc0-74aedd9378b7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26314739-180a-45be-9bc0-74aedd9378b7\",\n      \"name\": \"26314739-180a-45be-9bc0-74aedd9378b7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T00:42:44.968602571Z\",\n      \"endTime\": \"2026-08-21T01:06:45.158393217Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sakjl0nbdbh3dlb0ni55h84f7c279gs still exists (deletion dispatched at 2026-08-21T00:42:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012201b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-fnngq6qtshfw\" to finish deleting: GET https://rp.j8029568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26314739-180a-45be-9bc0-74aedd9378b7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/26314739-180a-45be-9bc0-74aedd9378b7\",\n      \"name\": \"26314739-180a-45be-9bc0-74aedd9378b7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T00:42:44.968602571Z\",\n      \"endTime\": \"2026-08-21T01:06:45.158393217Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sakjl0nbdbh3dlb0ni55h84f7c279gs still exists (deletion dispatched at 2026-08-21T00:42:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012ae180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-jflf6brl2gcq\" to finish deleting: GET https://rp.j2645888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0d2b3f4-a9c9-4fe3-9d6c-919fb891cf66\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0d2b3f4-a9c9-4fe3-9d6c-919fb891cf66\",\n      \"name\": \"b0d2b3f4-a9c9-4fe3-9d6c-919fb891cf66\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:41:11.57390019Z\",\n      \"endTime\": \"2026-08-19T16:05:19.863368819Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9ne21gf9tvn49fdnint8mmarsqg6fl still exists (deletion dispatched at 2026-08-19T15:41:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012ae180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-jflf6brl2gcq\" to finish deleting: GET https://rp.j2645888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0d2b3f4-a9c9-4fe3-9d6c-919fb891cf66\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0d2b3f4-a9c9-4fe3-9d6c-919fb891cf66\",\n      \"name\": \"b0d2b3f4-a9c9-4fe3-9d6c-919fb891cf66\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T15:41:11.57390019Z\",\n      \"endTime\": \"2026-08-19T16:05:19.863368819Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9ne21gf9tvn49fdnint8mmarsqg6fl still exists (deletion dispatched at 2026-08-19T15:41:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a92258\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-75kk7qpk9qxd\" to finish deleting: GET https://rp.j6737024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75303a92-754a-4e29-9094-d572e7177271\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75303a92-754a-4e29-9094-d572e7177271\",\n      \"name\": \"75303a92-754a-4e29-9094-d572e7177271\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T18:21:10.70474409Z\",\n      \"endTime\": \"2026-08-18T18:45:20.134803808Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s94vo54eitg07mhl2shtkmoaum52n2q still exists (deletion dispatched at 2026-08-18T18:21:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a92258\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-75kk7qpk9qxd\" to finish deleting: GET https://rp.j6737024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75303a92-754a-4e29-9094-d572e7177271\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/75303a92-754a-4e29-9094-d572e7177271\",\n      \"name\": \"75303a92-754a-4e29-9094-d572e7177271\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T18:21:10.70474409Z\",\n      \"endTime\": \"2026-08-18T18:45:20.134803808Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s94vo54eitg07mhl2shtkmoaum52n2q still exists (deletion dispatched at 2026-08-18T18:21:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090580629858029568","occurred_at":"2026-08-20T23:23:55Z","signature_id":"a688387e283d257bce8ddbe41e0174e2034ceb6121049d31c33a73bba2fccaa9","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090079394612645888","occurred_at":"2026-08-19T14:12:08Z","signature_id":"e22cca04f92354e3143765d7692aaa7085730c8c9dec1035cae8159d9630b31b","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089759198756737024","occurred_at":"2026-08-18T16:59:53Z","signature_id":"519c8ce6227540659bee65c3d400d130d9b1554c59aa43a959722dd3d4fef219","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708056525934592","occurred_at":"2026-08-18T13:36:10Z","signature_id":"72d3e904acd12f7e44a47b8c6556c62f3859c7f5c3a948adb211c8c51a93842a","pr_number":6585,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"429fcab9de847861bc73d647c851f740bbfe30154630193b6c0f1c86d0c6c3c8","failure_pattern":"alert [hcp] KubeDeploymentRolloutStuck fired","search_query":"alert [hcp] KubeDeploymentRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentRolloutStuck does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.6814310051107325,"anchor_occurrences":4,"after_last_push_count":12,"trend_counts":[4,3,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":48,"prior_last_seen_at":"2026-08-19T14:06:51Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentRolloutStuck does not fire","occurrences":4}],"full_error_samples":["alert fired 22 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:35:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h/capi-provider is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:35:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8-y7z1l5m6q1x6c2h/capi-provider is not progressing for longer than 30 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:36:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f/capi-provider is not progressing for longer than 30 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:36:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s/capi-provider is not progressing for longer than 30 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:36:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster/capi-provider is not progressing for longer than 30 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:36:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e/capi-provider is not progressing for longer than 30 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:36:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9ift45upic3366kju9079j8v3di0-g6r2f3w3i0o4c0e/capi-provider is not progressing for longer than 30 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:36:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9aful10q0muufmauvdqcggd5vuq6-d9u1y3o3g5s5u1f/capi-provider is not progressing for longer than 30 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:36:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9lffu4hp18ss0eii14a257i26gtd-v5i9e6m0f4x4o8s/capi-provider is not progressing for longer than 30 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:36:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9m6u6ptoc9e5qj33o34bcjr18f0p-oidc-wi-cluster/capi-provider is not progressing for longer than 30 minutes.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:38:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress/capi-provider is not progressing for longer than 30 minutes.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:38:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d/capi-provider is not progressing for longer than 30 minutes.\nFiring 13:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:38:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9iubs7q406q106e40lr7b2sp6op1-c7x1w1p2v5m7b7d/capi-provider is not progressing for longer than 30 minutes.\nFiring 14:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:38:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9ma79lm1io9l6ulgt0sbi5a82j44-private-ingress/capi-provider is not progressing for longer than 30 minutes.\nFiring 15:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:38:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r/capi-provider is not progressing for longer than 30 minutes.\nFiring 16:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:38:18Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9jehqg52aoq01j4epad5srqkbk4u-q8e9e4k3l8v2m8r/capi-provider is not progressing for longer than 30 minutes.\nFiring 17:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:40:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster/capi-provider is not progressing for longer than 30 minutes.\nFiring 18:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:40:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster/capi-provider is not progressing for longer than 30 minutes.\nFiring 19:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:40:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9na9ip4t9kt4mrqougpftn4e3ok60h-cilium-cluster/capi-provider is not progressing for longer than 30 minutes.\nFiring 20:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:40:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9nampntjoikfuh2rho0ein8iunlfpp-agg-cluster/capi-provider is not progressing for longer than 30 minutes.\nFiring 21:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T15:59:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q/capi-provider is not progressing for longer than 30 minutes.\nFiring 22:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:00:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0627968-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", namespace=\"ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-ctxwh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s9n9jjd3hb0tb4rfarl89p0v4d0q3ga-f8o3h2y6k4q5f9q/capi-provider is not progressing for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-18T16:58:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j4002688-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"multus-admission-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4002688/providers/microsoft.monitor/accounts/hcps-j4002688\", namespace=\"ocm-arohcpci01-2s92pk0u1ofvgc1m7g8huiv81anne9il-l7n0b6c5c7c4m8n\", pod=\"arohcp-monitor-kube-state-metrics-74c6bc74b7-9svnw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s92pk0u1ofvgc1m7g8huiv81anne9il-l7n0b6c5c7c4m8n/multus-admission-controller is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-18T17:00:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j4002688-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"multus-admission-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.183:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4002688/providers/microsoft.monitor/accounts/hcps-j4002688\", namespace=\"ocm-arohcpci01-2s92pk0u1ofvgc1m7g8huiv81anne9il-l7n0b6c5c7c4m8n\", pod=\"arohcp-monitor-kube-state-metrics-74c6bc74b7-9svnw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s92pk0u1ofvgc1m7g8huiv81anne9il-l7n0b6c5c7c4m8n/multus-admission-controller is not progressing for longer than 30 minutes.","alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-18T15:49:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0498304-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.171:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0498304/providers/microsoft.monitor/accounts/hcps-j0498304\", namespace=\"ocm-arohcpci01-2s92799ecc7up4q7o2em1q0eh9d69ooa-private-kas\", pod=\"arohcp-monitor-kube-state-metrics-74c6bc74b7-lmhjv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s92799ecc7up4q7o2em1q0eh9d69ooa-private-kas/capi-provider is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-18T15:49:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0498304-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.171:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0498304/providers/microsoft.monitor/accounts/hcps-j0498304\", namespace=\"ocm-arohcpci01-2s92799ecc7up4q7o2em1q0eh9d69ooa-private-kas\", pod=\"arohcp-monitor-kube-state-metrics-74c6bc74b7-lmhjv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s92799ecc7up4q7o2em1q0eh9d69ooa-private-kas/capi-provider is not progressing for longer than 30 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-18T15:49:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0498304-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.171:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0498304/providers/microsoft.monitor/accounts/hcps-j0498304\", namespace=\"ocm-arohcpci01-2s926uf9b5camq56g5afnepu3udl2sh8-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-74c6bc74b7-lmhjv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s926uf9b5camq56g5afnepu3udl2sh8-basic-cluster/capi-provider is not progressing for longer than 30 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-18T15:49:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0498304-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.171:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0498304/providers/microsoft.monitor/accounts/hcps-j0498304\", namespace=\"ocm-arohcpci01-2s926uf9b5camq56g5afnepu3udl2sh8-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-74c6bc74b7-lmhjv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment ocm-arohcpci01-2s926uf9b5camq56g5afnepu3udl2sh8-basic-cluster/capi-provider is not progressing for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"51e0c474703df02d6ae5c94d0202e9b971d8b5a564553aa7a9eec0fcfa04c3ac","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089723779814002688","occurred_at":"2026-08-18T14:39:04Z","signature_id":"ff2ea8f36f0b788bac3a873e71c686674cc6c1ead3c4617bb50cb492ba1a55dc","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089713360550498304","occurred_at":"2026-08-18T13:57:44Z","signature_id":"9251112ea65823b7f07d4d6e399677ab6710bda44e7db3c876807f349c09a2fa","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708056525934592","occurred_at":"2026-08-18T13:36:10Z","signature_id":"bcb5c2d8a5a6db75a06f54ef6f73342c5d252fbc446ddd0e20effef9e37cd140","pr_number":6585,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c4e2bd93d0410f48ddfa390cc77604492607a58b2a4eaff204baa6c7bed3af07","failure_pattern":"alert [svc] KubeContainerOOMKilled fired","search_query":"alert [svc] KubeContainerOOMKilled fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.6814310051107325,"anchor_occurrences":4,"after_last_push_count":2,"trend_counts":[0,3,0,0,0,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-18T19:17:13Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","occurrences":4}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-24T16:04:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-cxwt8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e03984bb-ef95-4df0-aec9-95c39b0e3c41\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-cxwt8 on cluster ci01-j9565440-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-24T16:04:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-jbtcb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1ec91183-7c07-473b-b1d1-9df325a61c81\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-jbtcb on cluster ci01-j9565440-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-24T16:04:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-jbtcb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1ec91183-7c07-473b-b1d1-9df325a61c81\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-jbtcb on cluster ci01-j9565440-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-24T16:04:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-cxwt8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e03984bb-ef95-4df0-aec9-95c39b0e3c41\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-cxwt8 on cluster ci01-j9565440-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-18T21:02:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j2509440-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.158:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2509440/providers/microsoft.monitor/accounts/services-j2509440\", namespace=\"arobit\", pod=\"arobit-forwarder-fcjgh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"OOMKilled\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ce013605-0558-41bb-98ab-a08ea5494672\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-fcjgh on cluster ci01-j2509440-mgmt-1 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-18T21:02:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j2509440-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.158:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2509440/providers/microsoft.monitor/accounts/services-j2509440\", namespace=\"arobit\", pod=\"arobit-forwarder-fcjgh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"OOMKilled\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ce013605-0558-41bb-98ab-a08ea5494672\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-fcjgh on cluster ci01-j2509440-mgmt-1 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-18T16:43:08Z\n  Ended:       2026-08-18T16:49:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j7122944-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.17:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7122944/providers/microsoft.monitor/accounts/services-j7122944\", namespace=\"arobit\", pod=\"arobit-forwarder-hmp22\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"OOMKilled\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"778afa21-902b-47da-bcf0-f52954ef307f\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-hmp22 on cluster ci01-j7122944-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-18T16:43:08Z\n  Ended:       2026-08-18T16:49:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j7122944-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.17:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7122944/providers/microsoft.monitor/accounts/services-j7122944\", namespace=\"arobit\", pod=\"arobit-forwarder-hmp22\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"OOMKilled\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"778afa21-902b-47da-bcf0-f52954ef307f\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-hmp22 on cluster ci01-j7122944-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"c239e70f5bf7ad0e42856cb2b9acc6d7c10a9ba7d6c9a4f5496cbd3609446018","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6589/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089793767912509440","occurred_at":"2026-08-18T19:17:13Z","signature_id":"90eff3914ac2e22e76d9f363c98e3f5d625d49f96dbcfac5afe58433fdd57349","pr_number":6589,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089740517767122944","occurred_at":"2026-08-18T15:45:39Z","signature_id":"fa4d88c9b57e656e5d371bc88b8125f1d357f00c608f6cc612b5da06b6a268e1","pr_number":4566,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089560445248606208","occurred_at":"2026-08-18T03:50:06Z","signature_id":"4fd9ee1f7cc8287c843624609f68cf47d4d8427e73eb0da764bba6a0d82acfd9","pr_number":6566,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a89f0d2edcd262580a6fd914503ced16ad52ae763f2c2255b4f5af55d375ecd3","failure_pattern":"alert [svc] KubeContainerWaiting fired","search_query":"alert [svc] KubeContainerWaiting fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerWaiting does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.6814310051107325,"anchor_occurrences":4,"after_last_push_count":1,"trend_counts":[0,0,2,1,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-21T03:34:17Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerWaiting does not fire","occurrences":4}],"full_error_samples":["alert fired 8 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T05:09:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j4800000-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"collector\", namespace=\"kube-system\", pod=\"eraser-aks-system-31747703-vmss000000-rwxmt\", severity=\"warning\"\n  Description: pod/eraser-aks-system-31747703-vmss000000-rwxmt in namespace kube-system on container collector has been in waiting state for longer than 1 hour.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T05:09:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j4800000-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"addon-token-adapter\", namespace=\"kube-system\", pod=\"ama-metrics-node-fnzb8\", severity=\"warning\"\n  Description: pod/ama-metrics-node-fnzb8 in namespace kube-system on container addon-token-adapter has been in waiting state for longer than 1 hour.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T05:09:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j4800000-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"prometheus-collector\", namespace=\"kube-system\", pod=\"ama-metrics-node-fnzb8\", severity=\"warning\"\n  Description: pod/ama-metrics-node-fnzb8 in namespace kube-system on container prometheus-collector has been in waiting state for longer than 1 hour.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T05:09:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j4800000-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"node-driver-registrar\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-w8ttz\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-w8ttz in namespace kube-system on container node-driver-registrar has been in waiting state for longer than 1 hour.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T05:09:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j4800000-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"secrets-store\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-w8ttz\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-w8ttz in namespace kube-system on container secrets-store has been in waiting state for longer than 1 hour.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T05:09:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j4800000-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"remover\", namespace=\"kube-system\", pod=\"eraser-aks-system-31747703-vmss000000-rwxmt\", severity=\"warning\"\n  Description: pod/eraser-aks-system-31747703-vmss000000-rwxmt in namespace kube-system on container remover has been in waiting state for longer than 1 hour.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T05:09:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j4800000-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"trivy-scanner\", namespace=\"kube-system\", pod=\"eraser-aks-system-31747703-vmss000000-rwxmt\", severity=\"warning\"\n  Description: pod/eraser-aks-system-31747703-vmss000000-rwxmt in namespace kube-system on container trivy-scanner has been in waiting state for longer than 1 hour.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T05:09:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j4800000-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"liveness-probe\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-w8ttz\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-w8ttz in namespace kube-system on container liveness-probe has been in waiting state for longer than 1 hour.","alert fired 8 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-20T14:39:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j2425216-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"node-driver-registrar\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-724fc\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-724fc in namespace kube-system on container node-driver-registrar has been in waiting state for longer than 1 hour.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-20T14:39:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j2425216-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"trivy-scanner\", namespace=\"kube-system\", pod=\"eraser-aks-system-12106338-vmss000000-vcv9l\", severity=\"warning\"\n  Description: pod/eraser-aks-system-12106338-vmss000000-vcv9l in namespace kube-system on container trivy-scanner has been in waiting state for longer than 1 hour.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-20T14:39:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j2425216-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"prometheus-collector\", namespace=\"kube-system\", pod=\"ama-metrics-node-7nn7h\", severity=\"warning\"\n  Description: pod/ama-metrics-node-7nn7h in namespace kube-system on container prometheus-collector has been in waiting state for longer than 1 hour.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-20T14:39:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j2425216-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"secrets-store\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-724fc\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-724fc in namespace kube-system on container secrets-store has been in waiting state for longer than 1 hour.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-08-20T14:40:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j2425216-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"liveness-probe\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-724fc\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-724fc in namespace kube-system on container liveness-probe has been in waiting state for longer than 1 hour.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-08-20T14:40:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j2425216-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"addon-token-adapter\", namespace=\"kube-system\", pod=\"ama-metrics-node-7nn7h\", severity=\"warning\"\n  Description: pod/ama-metrics-node-7nn7h in namespace kube-system on container addon-token-adapter has been in waiting state for longer than 1 hour.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-08-20T14:40:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j2425216-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"remover\", namespace=\"kube-system\", pod=\"eraser-aks-system-12106338-vmss000000-vcv9l\", severity=\"warning\"\n  Description: pod/eraser-aks-system-12106338-vmss000000-vcv9l in namespace kube-system on container remover has been in waiting state for longer than 1 hour.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-08-20T14:40:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j2425216-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"collector\", namespace=\"kube-system\", pod=\"eraser-aks-system-12106338-vmss000000-vcv9l\", severity=\"warning\"\n  Description: pod/eraser-aks-system-12106338-vmss000000-vcv9l in namespace kube-system on container collector has been in waiting state for longer than 1 hour.","alert fired 8 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T22:48:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6685312-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"secrets-store\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-xlp87\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-xlp87 in namespace kube-system on container secrets-store has been in waiting state for longer than 1 hour.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T22:48:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6685312-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"remover\", namespace=\"kube-system\", pod=\"eraser-aks-system-20685165-vmss000000-vp56v\", severity=\"warning\"\n  Description: pod/eraser-aks-system-20685165-vmss000000-vp56v in namespace kube-system on container remover has been in waiting state for longer than 1 hour.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T22:48:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6685312-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"collector\", namespace=\"kube-system\", pod=\"eraser-aks-system-20685165-vmss000000-vp56v\", severity=\"warning\"\n  Description: pod/eraser-aks-system-20685165-vmss000000-vp56v in namespace kube-system on container collector has been in waiting state for longer than 1 hour.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T22:48:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6685312-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"addon-token-adapter\", namespace=\"kube-system\", pod=\"ama-metrics-node-njm8m\", severity=\"warning\"\n  Description: pod/ama-metrics-node-njm8m in namespace kube-system on container addon-token-adapter has been in waiting state for longer than 1 hour.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T22:48:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6685312-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"liveness-probe\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-xlp87\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-xlp87 in namespace kube-system on container liveness-probe has been in waiting state for longer than 1 hour.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T22:48:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6685312-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"node-driver-registrar\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-xlp87\", severity=\"warning\"\n  Description: pod/aks-secrets-store-csi-driver-xlp87 in namespace kube-system on container node-driver-registrar has been in waiting state for longer than 1 hour.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T22:48:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6685312-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"trivy-scanner\", namespace=\"kube-system\", pod=\"eraser-aks-system-20685165-vmss000000-vp56v\", severity=\"warning\"\n  Description: pod/eraser-aks-system-20685165-vmss000000-vp56v in namespace kube-system on container trivy-scanner has been in waiting state for longer than 1 hour.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T22:48:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerWaiting\", cluster=\"ci01-j6685312-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"prometheus-collector\", namespace=\"kube-system\", pod=\"ama-metrics-node-njm8m\", severity=\"warning\"\n  Description: pod/ama-metrics-node-njm8m in namespace kube-system on container prometheus-collector has been in waiting state for longer than 1 hour."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090643626764800000","occurred_at":"2026-08-21T03:34:17Z","signature_id":"48b8e2cb57114629b4e6133fb41aeceaefffcf04b5ff1fc459dd655e0252babe","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090419688172425216","occurred_at":"2026-08-20T12:44:10Z","signature_id":"c3f4f642d5fe7b8cbf9f6c9b9e831f3a5a1bb8747828d07eeed3169305d122a4","pr_number":6617,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090180258316685312","occurred_at":"2026-08-19T20:53:01Z","signature_id":"8746d886253ada4663f87ba3097e3a4e02632ca623c664c63a686376132b6822","pr_number":6614,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090031884472946688","occurred_at":"2026-08-19T11:03:26Z","signature_id":"616e2d13a8d730313eb967f4b30c58b8a702753aad7be6756af4bbe5eefbe757","pr_number":6583,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":0.6814310051107325,"anchor_occurrences":4,"after_last_push_count":5,"trend_counts":[0,0,1,1,1,0,1,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":23,"prior_last_seen_at":"2026-08-23T03:43:19Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","occurrences":4}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-23T06:20:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j7940736-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.111:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7940736/providers/microsoft.monitor/accounts/services-j7940736\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-d5c7cc5b4-wxvwj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j7940736-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-23T06:20:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j7940736-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.111:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7940736/providers/microsoft.monitor/accounts/services-j7940736\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-d5c7cc5b4-wxvwj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j7940736-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T14:05:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j2827648-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.54:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2827648/providers/microsoft.monitor/accounts/services-j2827648\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-67688f45cb-sh5gb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j2827648-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-20T21:19:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j2508544-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2508544/providers/microsoft.monitor/accounts/services-j2508544\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-68dd4ccf65-vd77g\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j2508544-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-20T21:19:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KustoLogsDataStale\", cluster=\"ci01-j2508544-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.93:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2508544/providers/microsoft.monitor/accounts/services-j2508544\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-68dd4ccf65-vd77g\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n  Description: Kusto log data for table systemdlogs on cluster ci01-j2508544-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091370674797940736","occurred_at":"2026-08-23T03:43:19Z","signature_id":"f1da0e02131d79f9d743173dac73f72f60ddab0fc62aa516115c813c4011a54f","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6590/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090767289052827648","occurred_at":"2026-08-21T11:45:40Z","signature_id":"646de8fc5dad2d940ca1d6f85fbd453d41a6658308be00df36dea55382f0f44c","pr_number":6590,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"58c618af7602c927c3c11c14a7755006e83ca1016ea028f137f644807b446b83","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090079394612645888","occurred_at":"2026-08-19T14:12:08Z","signature_id":"1566a875249dafa2e6a8224e300d970974f75796f4a571cd360e9836b7e23199","pr_number":6601,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.5110732538330494,"anchor_occurrences":4,"after_last_push_count":1,"trend_counts":[0,0,2,2,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-20T19:00:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0017e29c0\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.33) failed: no version in clusterversion/version status.history is greater than initial \"4.20.33\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0017e29c0\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.33) failed: no version in clusterversion/version status.history is greater than initial \"4.20.33\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00120a5b8\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.28) failed: no version in clusterversion/version status.history is greater than initial \"4.21.28\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00120a5b8\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.28) failed: no version in clusterversion/version status.history is greater than initial \"4.21.28\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000883788\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.33) failed: no version in clusterversion/version status.history is greater than initial \"4.20.33\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000883788\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.33) failed: no version in clusterversion/version status.history is greater than initial \"4.20.33\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"1e125fa309bbfccb11249a0675fd870ebfb9e84ba21b179b4396c1b5598d7407","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090514291462508544","occurred_at":"2026-08-20T19:00:21Z","signature_id":"278e50384d73aa823a12869e8d59b4d21af4c3d554b9dfe27062a944fd0b322d","pr_number":6584,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"9cc8145e1eea121a8e0d1d77fe937aa6850e73bb34ffac96adf809bdcb76deb6","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"3ce6644c62e65c2c36bdbcc4461f08f990d2eba9f705a686f886c81cf4554eb5","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4a2f8bcf8d29c66434783789d75f6d18e6e94158f0e0f35e53c0353bce508345","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":4,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.34071550255536626,"anchor_occurrences":4,"after_last_push_count":3,"also_in":["STG"],"trend_counts":[1,0,2,0,0,0,0,2],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-19T18:19:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e70e20\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-wmkndtkx988t\" to finish creating: GET https://rp.j9565440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9d4baef-549e-4561-b748-439453d113af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9d4baef-549e-4561-b748-439453d113af\",\n      \"name\": \"a9d4baef-549e-4561-b748-439453d113af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:49:08.39871397Z\",\n      \"endTime\": \"2026-08-24T16:08:11.236941815Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e70e20\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-wmkndtkx988t\" to finish creating: GET https://rp.j9565440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9d4baef-549e-4561-b748-439453d113af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a9d4baef-549e-4561-b748-439453d113af\",\n      \"name\": \"a9d4baef-549e-4561-b748-439453d113af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:49:08.39871397Z\",\n      \"endTime\": \"2026-08-24T16:08:11.236941815Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000408ec0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-bn9cdfvxcqll\" to finish creating: GET https://rp.j9565440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75726762-be88-402f-a564-60c91125dc8b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75726762-be88-402f-a564-60c91125dc8b\",\n      \"name\": \"75726762-be88-402f-a564-60c91125dc8b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:47:28.10624954Z\",\n      \"endTime\": \"2026-08-24T16:06:31.149377863Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000408ec0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-bn9cdfvxcqll\" to finish creating: GET https://rp.j9565440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75726762-be88-402f-a564-60c91125dc8b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75726762-be88-402f-a564-60c91125dc8b\",\n      \"name\": \"75726762-be88-402f-a564-60c91125dc8b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:47:28.10624954Z\",\n      \"endTime\": \"2026-08-24T16:06:31.149377863Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e70880\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-x6prxbl8v6bk\" to finish creating: GET https://rp.j1797504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e2382ad9-6776-48ee-9bf9-e8c510952cb6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e2382ad9-6776-48ee-9bf9-e8c510952cb6\",\n      \"name\": \"e2382ad9-6776-48ee-9bf9-e8c510952cb6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:08:40.647335859Z\",\n      \"endTime\": \"2026-08-19T19:27:47.028573359Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e70880\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-x6prxbl8v6bk\" to finish creating: GET https://rp.j1797504.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e2382ad9-6776-48ee-9bf9-e8c510952cb6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e2382ad9-6776-48ee-9bf9-e8c510952cb6\",\n      \"name\": \"e2382ad9-6776-48ee-9bf9-e8c510952cb6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:08:40.647335859Z\",\n      \"endTime\": \"2026-08-19T19:27:47.028573359Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"7dd423d6ebe7dc246942b7ca078647aad56aaec8baea7f2d4aa8292a909f5f19","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"d07feba14b1358e1ba91e00ab3fcc20b89813cd03b2520364e81ecc050b70a83","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"5249410476dac4a69075b89f7eafcada175646a0b2ec9eb6125986a39843aef1","pr_number":6611,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090141674171797504","occurred_at":"2026-08-19T18:19:42Z","signature_id":"6cc6d9075dc9c54a3a94cfefb4e9f32ae722fa4876cd14876ed318bd9d803a1e","pr_number":6611,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d290d868b88709fe4f55b0fa5c26621f9cf2dfa1a679177187f94d52f4b5e00a","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initialVersion=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initialVersion=4","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,4,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-18T02:19:48Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:134]: Timed out after 2700.006s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00154daa0\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initialVersion=4.20.33, targetVersion=4.20.34) failed: cluster did not install to the correct install version. Expected: 4.20.33. Actual: 4.20.34\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:134]: Timed out after 2700.006s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00154daa0\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initialVersion=4.20.33, targetVersion=4.20.34) failed: cluster did not install to the correct install version. Expected: 4.20.33. Actual: 4.20.34\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:134]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001301200\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initialVersion=4.21.28, targetVersion=4.21.29) failed: cluster did not install to the correct install version. Expected: 4.21.28. Actual: 4.21.29\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:134]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001301200\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initialVersion=4.21.28, targetVersion=4.21.29) failed: cluster did not install to the correct install version. Expected: 4.21.28. Actual: 4.21.29\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:134]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00157f068\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initialVersion=4.22.8, targetVersion=4.22.9) failed: cluster did not install to the correct install version. Expected: 4.22.8. Actual: 4.22.9\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:134]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00157f068\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initialVersion=4.22.8, targetVersion=4.22.9) failed: cluster did not install to the correct install version. Expected: 4.22.8. Actual: 4.22.9\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089537718320107520","occurred_at":"2026-08-18T02:19:48Z","signature_id":"086271636c280e9c3ba8e5b414af29603e4bf504908a0341461b5ad7936a0e9c","pr_number":6240,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089537718320107520","occurred_at":"2026-08-18T02:19:48Z","signature_id":"5e9c184aac6af4c14bfb6c484da9a8c755a10c95086d7ac4c9657a6c499e6895","pr_number":6240,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089537718320107520","occurred_at":"2026-08-18T02:19:48Z","signature_id":"750f106cdf8be27ef1ada8e0d89d49967ead1a2a383c0b0c65972299906824f9","pr_number":6240,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6240/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089537718320107520","occurred_at":"2026-08-18T02:19:48Z","signature_id":"81aecfcfaa532f962505d81515a50a90196163b89f45e22c84502483a23bf2aa","pr_number":6240,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"392750d533ddeabf76d5209f9e2a09e5f48e64ee98fce8c13f1c6bd36af29cfa","failure_pattern":"Deserialization Error: no output from command","search_query":"Deserialization Error: no output from command","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.5110732538330494,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[1,3,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-18T10:46:28Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":3}],"full_error_samples":["[scheduler] pool mi-containers=60 (source: LEASED_MSI_CONTAINERS), 94 specs (5×0, 87×1, 2×2+)\n[scheduler] WARNING: test \"Engineering should be able to retrieve kusto logs for a cluster and services\" declares resource pool demands but no pool capacity is configured\n  Running Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1787053121 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Engineering should be able to retrieve kusto logs for a cluster and services [PreLaunchSetup:None, Importance:High, Positivity:Positive, Core-Infra-Service, Development-Only, ARO-HCP-RP-API-Compatible, MIContainers:1]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:34\n  \"ts\"=\"2026-08-18 11:38:41.084726\" \"level\"=0 \"msg\"=\"===== TEST CASE BEGAN =====\"\n  \"ts\"=\"2026-08-18 11:38:41.084775\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Engineering-should-be-able-to-retrieve-kusto-logs-for-a-cluster-and-services|pid:1404\"\n  \"ts\"=\"2026-08-18 11:38:41.099018\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating a resource group @ 08/18/26 11:38:41.099\n  \"ts\"=\"2026-08-18 11:38:41.099181\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"kusto-logs-n2bxlwcx6b6p\"\n    STEP: creating cluster parameters @ 08/18/26 11:38:43.103\n    STEP: creating customer resources @ 08/18/26 11:38:43.507\n    STEP: creating the HCP cluster @ 08/18/26 11:39:40.909\n  \"ts\"=\"2026-08-18 11:39:40.909394\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"engineering-hcp-cluster\" \"resourceGroup\"=\"kusto-logs-n2bxlwcx6b6p\" \"version\"=\"4.20.34\" \"channelGroup\"=\"candidate\"\n    STEP: verifying kusto logs are present @ 08/18/26 11:47:40.552\npanic: runtime error: invalid memory address or nil pointer dereference\n[signal SIGSEGV: segmentation violation code=0x1 addr=0x18 pc=0xb7d80d4]\n\ngoroutine 1138 [running]:\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0xc002209da0, {0xfb5d3d0, 0xc000dd0730}, {0xfb716a0, 0xc000fe7180}, 0xc001a8e0e0)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:128 +0x514\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x73\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:93 +0x50\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1100\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:78 +0x95\nCommand Error: exit status 2\nDeserialization Error: no output from command\n[scheduler] pool mi-containers=60 (source: LEASED_MSI_CONTAINERS), 94 specs (5×0, 87×1, 2×2+)\n[scheduler] WARNING: test \"Engineering should be able to retrieve kusto logs for a cluster and services\" declares resource pool demands but no pool capacity is configured\n  Running Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1787053121 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Engineering should be able to retrieve kusto logs for a cluster and services [PreLaunchSetup:None, Importance:High, Positivity:Positive, Core-Infra-Service, Development-Only, ARO-HCP-RP-API-Compatible, MIContainers:1]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:34\n  \"ts\"=\"2026-08-18 11:38:41.084726\" \"level\"=0 \"msg\"=\"===== TEST CASE BEGAN =====\"\n  \"ts\"=\"2026-08-18 11:38:41.084775\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Engineering-should-be-able-to-retrieve-kusto-logs-for-a-cluster-and-services|pid:1404\"\n  \"ts\"=\"2026-08-18 11:38:41.099018\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating a resource group @ 08/18/26 11:38:41.099\n  \"ts\"=\"2026-08-18 11:38:41.099181\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"kusto-logs-n2bxlwcx6b6p\"\n    STEP: creating cluster parameters @ 08/18/26 11:38:43.103\n    STEP: creating customer resources @ 08/18/26 11:38:43.507\n    STEP: creating the HCP cluster @ 08/18/26 11:39:40.909\n  \"ts\"=\"2026-08-18 11:39:40.909394\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"engineering-hcp-cluster\" \"resourceGroup\"=\"kusto-logs-n2bxlwcx6b6p\" \"version\"=\"4.20.34\" \"channelGroup\"=\"candidate\"\n    STEP: verifying kusto logs are present @ 08/18/26 11:47:40.552\npanic: runtime error: invalid memory address or nil pointer dereference\n[signal SIGSEGV: segmentation violation code=0x1 addr=0x18 pc=0xb7d80d4]\n\ngoroutine 1138 [running]:\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0xc002209da0, {0xfb5d3d0, 0xc000dd0730}, {0xfb716a0, 0xc000fe7180}, 0xc001a8e0e0)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:128 +0x514\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x73\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:93 +0x50\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1100\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:78 +0x95\nCommand Error: exit status 2\nDeserialization Error: no output from command","[scheduler] pool mi-containers=60 (source: LEASED_MSI_CONTAINERS), 94 specs (5×0, 87×1, 2×2+)\n[scheduler] WARNING: test \"Engineering should be able to retrieve kusto logs for a cluster and services\" declares resource pool demands but no pool capacity is configured\n  Running Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1787039952 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Engineering should be able to retrieve kusto logs for a cluster and services [PreLaunchSetup:None, Importance:High, Positivity:Positive, Core-Infra-Service, Development-Only, ARO-HCP-RP-API-Compatible, MIContainers:1]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:34\n  \"ts\"=\"2026-08-18 07:59:12.973910\" \"level\"=0 \"msg\"=\"===== TEST CASE BEGAN =====\"\n  \"ts\"=\"2026-08-18 07:59:12.973963\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Engineering-should-be-able-to-retrieve-kusto-logs-for-a-cluster-and-services|pid:1423\"\n  \"ts\"=\"2026-08-18 07:59:13.002203\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating a resource group @ 08/18/26 07:59:13.002\n  \"ts\"=\"2026-08-18 07:59:13.002397\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"kusto-logs-rvz5fj9ld5qr\"\n    STEP: creating cluster parameters @ 08/18/26 07:59:15.168\n    STEP: creating customer resources @ 08/18/26 07:59:15.469\n    STEP: creating the HCP cluster @ 08/18/26 08:00:42.31\n  \"ts\"=\"2026-08-18 08:00:42.310593\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"engineering-hcp-cluster\" \"resourceGroup\"=\"kusto-logs-rvz5fj9ld5qr\" \"version\"=\"4.20.34\" \"channelGroup\"=\"candidate\"\n    STEP: verifying kusto logs are present @ 08/18/26 08:12:07.746\npanic: runtime error: invalid memory address or nil pointer dereference\n[signal SIGSEGV: segmentation violation code=0x1 addr=0x18 pc=0xb7d80d4]\n\ngoroutine 706 [running]:\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0xc0007004a0, {0xfb5d3d0, 0xc00084f400}, {0xfb716a0, 0xc000fc58c0}, 0xc00147b730)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:128 +0x514\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x73\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:93 +0x50\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 685\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:78 +0x95\nCommand Error: exit status 2\nDeserialization Error: no output from command\n[scheduler] pool mi-containers=60 (source: LEASED_MSI_CONTAINERS), 94 specs (5×0, 87×1, 2×2+)\n[scheduler] WARNING: test \"Engineering should be able to retrieve kusto logs for a cluster and services\" declares resource pool demands but no pool capacity is configured\n  Running Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1787039952 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Engineering should be able to retrieve kusto logs for a cluster and services [PreLaunchSetup:None, Importance:High, Positivity:Positive, Core-Infra-Service, Development-Only, ARO-HCP-RP-API-Compatible, MIContainers:1]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:34\n  \"ts\"=\"2026-08-18 07:59:12.973910\" \"level\"=0 \"msg\"=\"===== TEST CASE BEGAN =====\"\n  \"ts\"=\"2026-08-18 07:59:12.973963\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Engineering-should-be-able-to-retrieve-kusto-logs-for-a-cluster-and-services|pid:1423\"\n  \"ts\"=\"2026-08-18 07:59:13.002203\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating a resource group @ 08/18/26 07:59:13.002\n  \"ts\"=\"2026-08-18 07:59:13.002397\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"kusto-logs-rvz5fj9ld5qr\"\n    STEP: creating cluster parameters @ 08/18/26 07:59:15.168\n    STEP: creating customer resources @ 08/18/26 07:59:15.469\n    STEP: creating the HCP cluster @ 08/18/26 08:00:42.31\n  \"ts\"=\"2026-08-18 08:00:42.310593\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"engineering-hcp-cluster\" \"resourceGroup\"=\"kusto-logs-rvz5fj9ld5qr\" \"version\"=\"4.20.34\" \"channelGroup\"=\"candidate\"\n    STEP: verifying kusto logs are present @ 08/18/26 08:12:07.746\npanic: runtime error: invalid memory address or nil pointer dereference\n[signal SIGSEGV: segmentation violation code=0x1 addr=0x18 pc=0xb7d80d4]\n\ngoroutine 706 [running]:\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0xc0007004a0, {0xfb5d3d0, 0xc00084f400}, {0xfb716a0, 0xc000fc58c0}, 0xc00147b730)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:128 +0x514\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x73\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:93 +0x50\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 685\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:78 +0x95\nCommand Error: exit status 2\nDeserialization Error: no output from command","[scheduler] pool mi-containers=60 (source: LEASED_MSI_CONTAINERS), 94 specs (5×0, 87×1, 2×2+)\n[scheduler] WARNING: test \"Engineering should be able to retrieve kusto logs for a cluster and services\" declares resource pool demands but no pool capacity is configured\n  Running Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1787016532 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Engineering should be able to retrieve kusto logs for a cluster and services [PreLaunchSetup:None, Importance:High, Positivity:Positive, Core-Infra-Service, Development-Only, ARO-HCP-RP-API-Compatible, MIContainers:1]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:34\n  \"ts\"=\"2026-08-18 01:28:52.967177\" \"level\"=0 \"msg\"=\"===== TEST CASE BEGAN =====\"\n  \"ts\"=\"2026-08-18 01:28:52.967214\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Engineering-should-be-able-to-retrieve-kusto-logs-for-a-cluster-and-services|pid:1413\"\n  \"ts\"=\"2026-08-18 01:28:52.980139\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating a resource group @ 08/18/26 01:28:52.98\n  \"ts\"=\"2026-08-18 01:28:52.980275\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"kusto-logs-h2xlqx47vsfl\"\n    STEP: creating cluster parameters @ 08/18/26 01:28:54.649\n    STEP: creating customer resources @ 08/18/26 01:28:54.943\n    STEP: creating the HCP cluster @ 08/18/26 01:30:02.242\n  \"ts\"=\"2026-08-18 01:30:02.242888\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"engineering-hcp-cluster\" \"resourceGroup\"=\"kusto-logs-h2xlqx47vsfl\" \"version\"=\"4.20.34\" \"channelGroup\"=\"candidate\"\n    STEP: verifying kusto logs are present @ 08/18/26 01:39:57.075\npanic: runtime error: invalid memory address or nil pointer dereference\n[signal SIGSEGV: segmentation violation code=0x1 addr=0x18 pc=0xb7d80d4]\n\ngoroutine 1104 [running]:\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0xc001be6ac0, {0xfb5d3d0, 0xc000cea640}, {0xfb716a0, 0xc000e9fb00}, 0xc0012f0460)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:128 +0x514\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x73\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:93 +0x50\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1098\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:78 +0x95\nCommand Error: exit status 2\nDeserialization Error: no output from command\n[scheduler] pool mi-containers=60 (source: LEASED_MSI_CONTAINERS), 94 specs (5×0, 87×1, 2×2+)\n[scheduler] WARNING: test \"Engineering should be able to retrieve kusto logs for a cluster and services\" declares resource pool demands but no pool capacity is configured\n  Running Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1787016532 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Engineering should be able to retrieve kusto logs for a cluster and services [PreLaunchSetup:None, Importance:High, Positivity:Positive, Core-Infra-Service, Development-Only, ARO-HCP-RP-API-Compatible, MIContainers:1]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:34\n  \"ts\"=\"2026-08-18 01:28:52.967177\" \"level\"=0 \"msg\"=\"===== TEST CASE BEGAN =====\"\n  \"ts\"=\"2026-08-18 01:28:52.967214\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Engineering-should-be-able-to-retrieve-kusto-logs-for-a-cluster-and-services|pid:1413\"\n  \"ts\"=\"2026-08-18 01:28:52.980139\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n    STEP: creating a resource group @ 08/18/26 01:28:52.98\n  \"ts\"=\"2026-08-18 01:28:52.980275\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"kusto-logs-h2xlqx47vsfl\"\n    STEP: creating cluster parameters @ 08/18/26 01:28:54.649\n    STEP: creating customer resources @ 08/18/26 01:28:54.943\n    STEP: creating the HCP cluster @ 08/18/26 01:30:02.242\n  \"ts\"=\"2026-08-18 01:30:02.242888\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"engineering-hcp-cluster\" \"resourceGroup\"=\"kusto-logs-h2xlqx47vsfl\" \"version\"=\"4.20.34\" \"channelGroup\"=\"candidate\"\n    STEP: verifying kusto logs are present @ 08/18/26 01:39:57.075\npanic: runtime error: invalid memory address or nil pointer dereference\n[signal SIGSEGV: segmentation violation code=0x1 addr=0x18 pc=0xb7d80d4]\n\ngoroutine 1104 [running]:\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto.(*Client).ExecutePreconfiguredQuery(0xc001be6ac0, {0xfb5d3d0, 0xc000cea640}, {0xfb716a0, 0xc000e9fb00}, 0xc0012f0460)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/kusto/client.go:128 +0x514\ngithub.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather.(*QueryClient).ConcurrentQueries.func1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/tooling/hcpctl/pkg/mustgather/queryclient.go:68 +0x73\ngolang.org/x/sync/errgroup.(*Group).Go.func1()\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:93 +0x50\ncreated by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1098\n\t/go/pkg/mod/golang.org/x/sync@v0.22.0/errgroup/errgroup.go:78 +0x95\nCommand Error: exit status 2\nDeserialization Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089665224855523328","occurred_at":"2026-08-18T10:46:28Z","signature_id":"fd753702df6762a4686987e9c2bbacdbd5e498443dedfe86df378247b78dc643","pr_number":6573,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089615938193199104","occurred_at":"2026-08-18T07:30:25Z","signature_id":"7bfcbbe8e28e2e108b2e3c3c3af4fd57e7b74357f73418aa3230fb2792bb8938","pr_number":6573,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089512043190161408","occurred_at":"2026-08-18T00:37:46Z","signature_id":"c2f13b13afe01e53820bc47f18c78de603f2fd538a93d7f65d154df3b6991b8c","pr_number":6573,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1e8fa14fe9270d4b471317a8bfdf5ad8e4d14e7d6f78446e573ecf2d55d134b4","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.5110732538330494,"anchor_occurrences":3,"after_last_push_count":1,"also_in":["PROD","STG"],"trend_counts":[0,3,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-18T18:57:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b8ac00\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-nl2f7vpzzf7s\" to finish creating: GET https://rp.j5683584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8bdf22cc-8ae5-4bc6-8e44-de1dc8c990fe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8bdf22cc-8ae5-4bc6-8e44-de1dc8c990fe\",\n      \"name\": \"8bdf22cc-8ae5-4bc6-8e44-de1dc8c990fe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T19:40:35.455444109Z\",\n      \"endTime\": \"2026-08-18T19:59:44.020810372Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b8ac00\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-nl2f7vpzzf7s\" to finish creating: GET https://rp.j5683584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8bdf22cc-8ae5-4bc6-8e44-de1dc8c990fe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8bdf22cc-8ae5-4bc6-8e44-de1dc8c990fe\",\n      \"name\": \"8bdf22cc-8ae5-4bc6-8e44-de1dc8c990fe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T19:40:35.455444109Z\",\n      \"endTime\": \"2026-08-18T19:59:44.020810372Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00094c560\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-r2dw8splnxfc\" to finish creating: GET https://rp.j6278912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8ab0a725-5e61-4cfc-a3ab-0e37763c714c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8ab0a725-5e61-4cfc-a3ab-0e37763c714c\",\n      \"name\": \"8ab0a725-5e61-4cfc-a3ab-0e37763c714c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T19:40:01.462032744Z\",\n      \"endTime\": \"2026-08-18T19:59:04.936468722Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00094c560\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-r2dw8splnxfc\" to finish creating: GET https://rp.j6278912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8ab0a725-5e61-4cfc-a3ab-0e37763c714c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8ab0a725-5e61-4cfc-a3ab-0e37763c714c\",\n      \"name\": \"8ab0a725-5e61-4cfc-a3ab-0e37763c714c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T19:40:01.462032744Z\",\n      \"endTime\": \"2026-08-18T19:59:04.936468722Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000880a60\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mx8bhmtc2bt4\" to finish creating: GET https://rp.j4461568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ac828024-ca50-4ce2-9064-9688a53bad69\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ac828024-ca50-4ce2-9064-9688a53bad69\",\n      \"name\": \"ac828024-ca50-4ce2-9064-9688a53bad69\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T15:34:28.369801816Z\",\n      \"endTime\": \"2026-08-18T15:53:34.581208912Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000880a60\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mx8bhmtc2bt4\" to finish creating: GET https://rp.j4461568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ac828024-ca50-4ce2-9064-9688a53bad69\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ac828024-ca50-4ce2-9064-9688a53bad69\",\n      \"name\": \"ac828024-ca50-4ce2-9064-9688a53bad69\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T15:34:28.369801816Z\",\n      \"endTime\": \"2026-08-18T15:53:34.581208912Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6558/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089788762895683584","occurred_at":"2026-08-18T18:57:22Z","signature_id":"8b302d19853ff0abb5a7ff1cb740d670f1934fd38513e8b630ea9ec6dc329ee1","pr_number":6558,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089786162666278912","occurred_at":"2026-08-18T18:47:02Z","signature_id":"bd6c9af2a3e89993c190c2980adc03b8824f1dfda4ef839ad34a825a250ee473","pr_number":6157,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6564/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725877364461568","occurred_at":"2026-08-18T14:47:25Z","signature_id":"4e4b984f1ea512ccebf792fd5a20a5054bff5eebd515861f18941ed25551208c","pr_number":6564,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"e42c684cbff8153f648b830fb145663fcbe7dcbcc9111d106802aa7f03e55e71","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.5110732538330494,"anchor_occurrences":3,"after_last_push_count":0,"also_in":["STG"],"trend_counts":[0,1,0,1,0,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-20T17:48:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001580bc0\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-cbbn9kpl7fxv\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c6f112da-48e3-4cbb-a1b9-a47159e85ff5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c6f112da-48e3-4cbb-a1b9-a47159e85ff5\",\n      \"name\": \"c6f112da-48e3-4cbb-a1b9-a47159e85ff5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:18:01.504032647Z\",\n      \"endTime\": \"2026-08-24T15:37:08.017891025Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, azure-cloud-controller-manager, cluster-node-tuning-operator, olm-operator, olm-collect-profiles, cluster-autoscaler, control-plane-pki-operator, kube-controller-manager, community-operators-catalog, redhat-operators-catalog, machine-approver, openshift-controller-manager, cluster-storage-operator, dns-operator, redhat-marketplace-catalog, cluster-version-operator, openshift-route-controller-manager, ingress-operator, kube-scheduler, cluster-image-registry-operator, konnectivity-agent, catalog-operator, cluster-network-operator, ignition-server, ignition-server-proxy, openshift-apiserver, hosted-cluster-config-operator, csi-snapshot-controller-operator, packageserver, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001580bc0\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-cbbn9kpl7fxv\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c6f112da-48e3-4cbb-a1b9-a47159e85ff5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c6f112da-48e3-4cbb-a1b9-a47159e85ff5\",\n      \"name\": \"c6f112da-48e3-4cbb-a1b9-a47159e85ff5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:18:01.504032647Z\",\n      \"endTime\": \"2026-08-24T15:37:08.017891025Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, azure-cloud-controller-manager, cluster-node-tuning-operator, olm-operator, olm-collect-profiles, cluster-autoscaler, control-plane-pki-operator, kube-controller-manager, community-operators-catalog, redhat-operators-catalog, machine-approver, openshift-controller-manager, cluster-storage-operator, dns-operator, redhat-marketplace-catalog, cluster-version-operator, openshift-route-controller-manager, ingress-operator, kube-scheduler, cluster-image-registry-operator, konnectivity-agent, catalog-operator, cluster-network-operator, ignition-server, ignition-server-proxy, openshift-apiserver, hosted-cluster-config-operator, csi-snapshot-controller-operator, packageserver, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-ptz96v\" with version 4.21.28 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001880ae0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-ptz96v: failed waiting for cluster=\"cluster-zstream-4-21-ptz96v\" in resourcegroup=\"rg-zstream-upgrade-4-21-ccgv7gb5sm22\" to finish creating: GET https://rp.j8125696.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2fbdf9d1-a05c-4017-9c28-571dc623c672\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2fbdf9d1-a05c-4017-9c28-571dc623c672\",\n      \"name\": \"2fbdf9d1-a05c-4017-9c28-571dc623c672\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T18:37:36.452911298Z\",\n      \"endTime\": \"2026-08-20T18:56:41.515494429Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, cluster-network-operator, certified-operators-catalog, ingress-operator, cluster-image-registry-operator, cluster-autoscaler, cluster-node-tuning-operator, dns-operator, konnectivity-agent, redhat-marketplace-catalog, kube-controller-manager, openshift-controller-manager, csi-snapshot-controller-operator, ignition-server-proxy, olm-operator, olm-collect-profiles, control-plane-pki-operator, cluster-storage-operator, catalog-operator, kube-scheduler, openshift-apiserver, openshift-route-controller-manager, redhat-operators-catalog, machine-approver, cluster-version-operator, hosted-cluster-config-operator, azure-cloud-controller-manager, ignition-server, packageserver, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-ptz96v\" with version 4.21.28 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001880ae0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-ptz96v: failed waiting for cluster=\"cluster-zstream-4-21-ptz96v\" in resourcegroup=\"rg-zstream-upgrade-4-21-ccgv7gb5sm22\" to finish creating: GET https://rp.j8125696.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2fbdf9d1-a05c-4017-9c28-571dc623c672\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2fbdf9d1-a05c-4017-9c28-571dc623c672\",\n      \"name\": \"2fbdf9d1-a05c-4017-9c28-571dc623c672\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T18:37:36.452911298Z\",\n      \"endTime\": \"2026-08-20T18:56:41.515494429Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, cluster-network-operator, certified-operators-catalog, ingress-operator, cluster-image-registry-operator, cluster-autoscaler, cluster-node-tuning-operator, dns-operator, konnectivity-agent, redhat-marketplace-catalog, kube-controller-manager, openshift-controller-manager, csi-snapshot-controller-operator, ignition-server-proxy, olm-operator, olm-collect-profiles, control-plane-pki-operator, cluster-storage-operator, catalog-operator, kube-scheduler, openshift-apiserver, openshift-route-controller-manager, redhat-operators-catalog, machine-approver, cluster-version-operator, hosted-cluster-config-operator, azure-cloud-controller-manager, ignition-server, packageserver, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1c4a0\u003e: \n    failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-kznjdfdlm5vz\" to finish creating: GET https://rp.j7122944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b49e9ba3-2d03-4f2f-b0d8-a6d6a9c6f5eb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b49e9ba3-2d03-4f2f-b0d8-a6d6a9c6f5eb\",\n      \"name\": \"b49e9ba3-2d03-4f2f-b0d8-a6d6a9c6f5eb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T16:29:33.767295492Z\",\n      \"endTime\": \"2026-08-18T16:48:39.984850743Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: azure-cloud-controller-manager, dns-operator, community-operators-catalog, cluster-policy-controller, machine-approver, cluster-version-operator, openshift-controller-manager, ignition-server-proxy, olm-operator, olm-collect-profiles, redhat-operators-catalog, cluster-autoscaler, kube-scheduler, cluster-network-operator, kube-storage-version-migrator, packageserver, konnectivity-agent, ignition-server, ingress-operator, cluster-image-registry-operator, control-plane-pki-operator, kube-controller-manager, openshift-route-controller-manager, cluster-storage-operator, csi-snapshot-controller-operator, catalog-operator, hosted-cluster-config-operator, cluster-node-tuning-operator, openshift-apiserver, redhat-marketplace-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1c4a0\u003e: \n    failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-kznjdfdlm5vz\" to finish creating: GET https://rp.j7122944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b49e9ba3-2d03-4f2f-b0d8-a6d6a9c6f5eb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b49e9ba3-2d03-4f2f-b0d8-a6d6a9c6f5eb\",\n      \"name\": \"b49e9ba3-2d03-4f2f-b0d8-a6d6a9c6f5eb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T16:29:33.767295492Z\",\n      \"endTime\": \"2026-08-18T16:48:39.984850743Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: azure-cloud-controller-manager, dns-operator, community-operators-catalog, cluster-policy-controller, machine-approver, cluster-version-operator, openshift-controller-manager, ignition-server-proxy, olm-operator, olm-collect-profiles, redhat-operators-catalog, cluster-autoscaler, kube-scheduler, cluster-network-operator, kube-storage-version-migrator, packageserver, konnectivity-agent, ignition-server, ingress-operator, cluster-image-registry-operator, control-plane-pki-operator, kube-controller-manager, openshift-route-controller-manager, cluster-storage-operator, csi-snapshot-controller-operator, catalog-operator, hosted-cluster-config-operator, cluster-node-tuning-operator, openshift-apiserver, redhat-marketplace-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"14a501f15a848fe949827032cbe7eab1732753bec28bb5cacc0d3876ab277f28","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"646c9d10959bdcfabf83f428c38022cd3ddc2083f56da46a37cd6ed226488e71","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089740517767122944","occurred_at":"2026-08-18T15:45:39Z","signature_id":"7e098fb45f691761a11c6d2426e3bc6f03612025578d74c3d4306e6776716713","pr_number":4566,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a6feac7cf53c211d52cb06ad454e22f897487b0c8e02adbc06fc4cde2c2fa6ef","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.5110732538330494,"anchor_occurrences":3,"after_last_push_count":4,"also_in":["STG"],"trend_counts":[2,2,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":10,"prior_last_seen_at":"2026-08-19T18:39:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a528a0\u003e: \n    GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ac96a8d5-d98c-41c2-a256-36b6060bad88\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ac96a8d5-d98c-41c2-a256-36b6060bad88\",\n      \"name\": \"ac96a8d5-d98c-41c2-a256-36b6060bad88\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:29:26.524288393Z\",\n      \"endTime\": \"2026-08-19T19:48:29.811530415Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, ingress-operator, csi-snapshot-controller-operator, packageserver, openshift-controller-manager, olm-operator, dns-operator, cluster-storage-operator, hosted-cluster-config-operator, cluster-network-operator, cluster-version-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a528a0\u003e: \n    GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ac96a8d5-d98c-41c2-a256-36b6060bad88\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ac96a8d5-d98c-41c2-a256-36b6060bad88\",\n      \"name\": \"ac96a8d5-d98c-41c2-a256-36b6060bad88\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:29:26.524288393Z\",\n      \"endTime\": \"2026-08-19T19:48:29.811530415Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, ingress-operator, csi-snapshot-controller-operator, packageserver, openshift-controller-manager, olm-operator, dns-operator, cluster-storage-operator, hosted-cluster-config-operator, cluster-network-operator, cluster-version-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-21-bs2lkklxljwq/cluster-candidate-4-21-hxfdst should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bd7520\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-hxfdst: failed waiting for cluster=\"cluster-candidate-4-21-hxfdst\" in resourcegroup=\"rg-candidate-4-21-bs2lkklxljwq\" to finish creating: GET https://rp.j9343872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db06c758-c8db-4fbd-9438-a269382b691e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db06c758-c8db-4fbd-9438-a269382b691e\",\n      \"name\": \"db06c758-c8db-4fbd-9438-a269382b691e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T20:19:08.987945021Z\",\n      \"endTime\": \"2026-08-18T20:38:15.340514803Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, csi-snapshot-controller-operator, olm-operator, cluster-version-operator, community-operators-catalog, hosted-cluster-config-operator, ingress-operator, catalog-operator, packageserver, redhat-operators-catalog, redhat-marketplace-catalog, cluster-network-operator, dns-operator, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-21-bs2lkklxljwq/cluster-candidate-4-21-hxfdst should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bd7520\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-hxfdst: failed waiting for cluster=\"cluster-candidate-4-21-hxfdst\" in resourcegroup=\"rg-candidate-4-21-bs2lkklxljwq\" to finish creating: GET https://rp.j9343872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db06c758-c8db-4fbd-9438-a269382b691e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db06c758-c8db-4fbd-9438-a269382b691e\",\n      \"name\": \"db06c758-c8db-4fbd-9438-a269382b691e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T20:19:08.987945021Z\",\n      \"endTime\": \"2026-08-18T20:38:15.340514803Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, csi-snapshot-controller-operator, olm-operator, cluster-version-operator, community-operators-catalog, hosted-cluster-config-operator, ingress-operator, catalog-operator, packageserver, redhat-operators-catalog, redhat-marketplace-catalog, cluster-network-operator, dns-operator, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c8da60\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-q7bzmdhm7zkd\" to finish creating: GET https://rp.j6278912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81876531-cc7c-4b96-8633-9c09cb6aa105\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81876531-cc7c-4b96-8633-9c09cb6aa105\",\n      \"name\": \"81876531-cc7c-4b96-8633-9c09cb6aa105\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T19:40:45.375485647Z\",\n      \"endTime\": \"2026-08-18T19:59:46.212136225Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, catalog-operator, hosted-cluster-config-operator, cluster-network-operator, redhat-operators-catalog, redhat-marketplace-catalog, konnectivity-agent, olm-operator, certified-operators-catalog, dns-operator, ingress-operator, cluster-storage-operator, csi-snapshot-controller-operator, packageserver, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c8da60\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-q7bzmdhm7zkd\" to finish creating: GET https://rp.j6278912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81876531-cc7c-4b96-8633-9c09cb6aa105\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81876531-cc7c-4b96-8633-9c09cb6aa105\",\n      \"name\": \"81876531-cc7c-4b96-8633-9c09cb6aa105\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T19:40:45.375485647Z\",\n      \"endTime\": \"2026-08-18T19:59:46.212136225Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, catalog-operator, hosted-cluster-config-operator, cluster-network-operator, redhat-operators-catalog, redhat-marketplace-catalog, konnectivity-agent, olm-operator, certified-operators-catalog, dns-operator, ingress-operator, cluster-storage-operator, csi-snapshot-controller-operator, packageserver, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 2 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"abd84fcd86a93cb5f5771dc01297e0cdabb7c3c66cb8870add8499b52e97b83e","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089797141739343872","occurred_at":"2026-08-18T19:30:39Z","signature_id":"d5661f5759197664aff9a19265117b7eafd8db0768b8199cae561f928c272219","pr_number":6588,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089786162666278912","occurred_at":"2026-08-18T18:47:02Z","signature_id":"2b8fc6d1d5901762918867c0e67a6d1edbdf27c2738d215daf30cbe1e21a685a","pr_number":6157,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.5110732538330494,"anchor_occurrences":3,"after_last_push_count":3,"also_in":["PROD","STG"],"trend_counts":[0,0,2,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-21T15:05:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001432360\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-7mnb7f8t2fc8\" to finish deleting: GET https://rp.j7310848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11242fc1-d50f-4f32-8786-9a40f8f52a03\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11242fc1-d50f-4f32-8786-9a40f8f52a03\",\n      \"name\": \"11242fc1-d50f-4f32-8786-9a40f8f52a03\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T16:20:06.708120379Z\",\n      \"endTime\": \"2026-08-21T16:44:15.494786631Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sb2fn1d5o4j56em902hb6rtk1gokehg still exists (deletion dispatched at 2026-08-21T16:20:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001432360\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-7mnb7f8t2fc8\" to finish deleting: GET https://rp.j7310848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11242fc1-d50f-4f32-8786-9a40f8f52a03\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/11242fc1-d50f-4f32-8786-9a40f8f52a03\",\n      \"name\": \"11242fc1-d50f-4f32-8786-9a40f8f52a03\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T16:20:06.708120379Z\",\n      \"endTime\": \"2026-08-21T16:44:15.494786631Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sb2fn1d5o4j56em902hb6rtk1gokehg still exists (deletion dispatched at 2026-08-21T16:20:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000dae120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-qcdkcr7lr97v\" to finish deleting: GET https://rp.j8557184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cbd3478f-c13b-4294-a2b4-aab3f405e864\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cbd3478f-c13b-4294-a2b4-aab3f405e864\",\n      \"name\": \"cbd3478f-c13b-4294-a2b4-aab3f405e864\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:47:36.049590288Z\",\n      \"endTime\": \"2026-08-19T20:11:36.820098818Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9rcmk3lenkuv9o0um7oqo6b42lnt24 still exists (deletion dispatched at 2026-08-19T19:47:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000dae120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-qcdkcr7lr97v\" to finish deleting: GET https://rp.j8557184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cbd3478f-c13b-4294-a2b4-aab3f405e864\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cbd3478f-c13b-4294-a2b4-aab3f405e864\",\n      \"name\": \"cbd3478f-c13b-4294-a2b4-aab3f405e864\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:47:36.049590288Z\",\n      \"endTime\": \"2026-08-19T20:11:36.820098818Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9rcmk3lenkuv9o0um7oqo6b42lnt24 still exists (deletion dispatched at 2026-08-19T19:47:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00178a108\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listqhsbzr\" in resourcegroup=\"ea-list-g46jmv84xvpr\" to finish deleting: GET https://rp.j1892352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cad9ef3-37df-488f-a560-fd20fe424e95\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cad9ef3-37df-488f-a560-fd20fe424e95\",\n      \"name\": \"9cad9ef3-37df-488f-a560-fd20fe424e95\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T09:22:58.762011919Z\",\n      \"endTime\": \"2026-08-19T09:47:05.74463192Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9i9kurs6ddnp8prrnr16efttcjng43 still exists (deletion dispatched at 2026-08-19T09:22:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00178a108\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listqhsbzr\" in resourcegroup=\"ea-list-g46jmv84xvpr\" to finish deleting: GET https://rp.j1892352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cad9ef3-37df-488f-a560-fd20fe424e95\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9cad9ef3-37df-488f-a560-fd20fe424e95\",\n      \"name\": \"9cad9ef3-37df-488f-a560-fd20fe424e95\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T09:22:58.762011919Z\",\n      \"endTime\": \"2026-08-19T09:47:05.74463192Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9i9kurs6ddnp8prrnr16efttcjng43 still exists (deletion dispatched at 2026-08-19T09:22:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090817534927310848","occurred_at":"2026-08-21T15:05:19Z","signature_id":"a489e8aa78d68850b06367b29a4da923ea7dc01f7a8e25f8f3561383918d490f","pr_number":6070,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090148755918557184","occurred_at":"2026-08-19T18:47:51Z","signature_id":"55ed54e871e5ba9e79f7a10853e03aea5541e1178fbf8962dee673333f8ced14","pr_number":6608,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089990952641892352","occurred_at":"2026-08-19T08:20:47Z","signature_id":"8bc6366d372ee7089c2b2207c73b9df938b4edc215900845a3fa3a5deb9f14de","pr_number":6538,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"281d6f2e10296c7900f78767c52e72d480329fd4f2572568eaae28e31a3fe696","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.f4w8s2m3g3e4n8e.cpdq.j7223552.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.f4w8s2m3g3e4n8e.cpdq.j7223552.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.131.82.100:48700-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.5110732538330494,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,1,2,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-10","2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-20T17:02:48Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:137]: failed to verify basic cluster health for idms-e2e-hcp-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc0010a5830\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.f4w8s2m3g3e4n8e.cpdq.j7223552.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.f4w8s2m3g3e4n8e.cpdq.j7223552.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.131.82.100:48700-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.f4w8s2m3g3e4n8e.cpdq.j7223552.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.f4w8s2m3g3e4n8e.cpdq.j7223552.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.131.82.100:48700-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:137]: failed to verify basic cluster health for idms-e2e-hcp-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc0010a5830\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.f4w8s2m3g3e4n8e.cpdq.j7223552.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.f4w8s2m3g3e4n8e.cpdq.j7223552.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.131.82.100:48700-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.f4w8s2m3g3e4n8e.cpdq.j7223552.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.f4w8s2m3g3e4n8e.cpdq.j7223552.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.131.82.100:48700-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:241]: failed to verify HCP cluster viability\nUnexpected error:\n    \u003c*errors.joinError | 0xc00113a678\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.f8q0u2e6o9u3o4v.60na.j3674112.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.f8q0u2e6o9u3o4v.60na.j3674112.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.130.38.9:35166-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.f8q0u2e6o9u3o4v.60na.j3674112.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.f8q0u2e6o9u3o4v.60na.j3674112.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.130.38.9:35166-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:241]: failed to verify HCP cluster viability\nUnexpected error:\n    \u003c*errors.joinError | 0xc00113a678\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.f8q0u2e6o9u3o4v.60na.j3674112.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.f8q0u2e6o9u3o4v.60na.j3674112.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.130.38.9:35166-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.f8q0u2e6o9u3o4v.60na.j3674112.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.f8q0u2e6o9u3o4v.60na.j3674112.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.130.38.9:35166-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:135]: failed to verify HCP cluster \"cluster-candidate-5-0-6b6mzt\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0xc000f25020\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.v7y4d9w4j7z1d4r.ptmj.j2752384.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.v7y4d9w4j7z1d4r.ptmj.j2752384.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.128.36.17:43656-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.v7y4d9w4j7z1d4r.ptmj.j2752384.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.v7y4d9w4j7z1d4r.ptmj.j2752384.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.128.36.17:43656-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:135]: failed to verify HCP cluster \"cluster-candidate-5-0-6b6mzt\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0xc000f25020\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.v7y4d9w4j7z1d4r.ptmj.j2752384.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.v7y4d9w4j7z1d4r.ptmj.j2752384.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.128.36.17:43656-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.v7y4d9w4j7z1d4r.ptmj.j2752384.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.v7y4d9w4j7z1d4r.ptmj.j2752384.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.128.36.17:43656-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090484744067223552","occurred_at":"2026-08-20T17:02:48Z","signature_id":"7648b78127bfdef7b2e7f4fb0449c332b1c8467f57775d688c9bd8ee54b5f2b4","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248642563674112","occurred_at":"2026-08-20T01:24:45Z","signature_id":"e9186488e62d93403aa4b53aad2c98b53df0c580027391c25bdf01e79761323c","pr_number":6615,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090087196852752384","occurred_at":"2026-08-19T14:43:14Z","signature_id":"8449f561e1d8c267e5688dd44ff0e20f4db56086e1dd6074eee4fe043bf86004","pr_number":6608,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.5110732538330494,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,2],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-22T00:32:00Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":3}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T17:55:35Z\n  Ended:       2026-08-24T18:27:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6361088-svc\", component=\"frontend\", method=\"put\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6361088-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T17:21:19Z\n  Ended:       2026-08-24T17:40:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9342336-svc\", component=\"frontend\", method=\"put\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9342336-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-22T01:28:34Z\n  Ended:       2026-08-22T02:01:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2714112-svc\", component=\"frontend\", method=\"put\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2714112-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091932359686361088","occurred_at":"2026-08-24T16:54:34Z","signature_id":"a3d3dfe16c45487c7de4833dcff7c72d504e34a1b391992ceada6266956233ad","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091921367619342336","occurred_at":"2026-08-24T16:10:41Z","signature_id":"d4e52f6e7208e4a023c43c3ea9a95264341c256a3dcba5d703516c2b2c81f97b","pr_number":6638,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090960143322714112","occurred_at":"2026-08-22T00:32:00Z","signature_id":"41cd7d7000b0d0b522b6e180f5feaf065d54e9a6666f9085acc644c179c02e0c","pr_number":6639,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.5110732538330494,"anchor_occurrences":3,"after_last_push_count":8,"trend_counts":[1,1,0,0,2,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":91,"prior_last_seen_at":"2026-08-21T20:12:06Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":3}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T20:56:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3460096-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"maestro\", pod=\"maestro-agent-666f6c69f9-jfc5v\", severity=\"warning\"\n  Description: Pod maestro/maestro-agent-666f6c69f9-jfc5v has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T02:42:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6439552-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-7c5f797746-w2wz8\", severity=\"warning\"\n  Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-7c5f797746-w2wz8 has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-18T23:55:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0124416-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"maestro\", pod=\"maestro-agent-6b55cfd9cc-dvzv8\", severity=\"warning\"\n  Description: Pod maestro/maestro-agent-6b55cfd9cc-dvzv8 has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894760083460096","occurred_at":"2026-08-21T20:12:06Z","signature_id":"271bd166fd209036f953ac22fa2a76816c81cef505fa3146bce1d1786a3f3845","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"9855d4e08bca12aa1036cc00d95206a600dd2efa6c513fbdd3e506c0bb255560","pr_number":6617,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089852994920124416","occurred_at":"2026-08-18T23:12:36Z","signature_id":"bfa4cab5738723a6b15f5946e22a199333c7f324fabd5696c7000a725bbfb10d","pr_number":6128,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"784380ace55b355f47065469da48bc1c7b73547d879efb7edb5ed96de913234e","failure_pattern":"couldn't get current server API group list: the server has asked for the client to provide credentials","search_query":"couldn't get current server API group list: the server has asked for the client to provide credentials","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.5110732538330494,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[1,1,0,1,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":11,"prior_last_seen_at":"2026-08-21T01:30:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:347]: Timed out after 300.004s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:332 with:\nkubectl should execute successfully from authorized VM with OIDC token\nUnexpected error:\n    \u003c*errors.errorString | 0xc000c401b0\u003e: \n    E0821 02:46:28.623242    2548 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0821 02:46:28.657456    2548 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0821 02:46:28.675878    2548 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0821 02:46:28.697350    2548 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0821 02:46:28.710066    2548 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    error: You must be logged in to the server (the server has asked for the client to provide credentials)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:347]: Timed out after 300.004s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:332 with:\nkubectl should execute successfully from authorized VM with OIDC token\nUnexpected error:\n    \u003c*errors.errorString | 0xc000c401b0\u003e: \n    E0821 02:46:28.623242    2548 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0821 02:46:28.657456    2548 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0821 02:46:28.675878    2548 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0821 02:46:28.697350    2548 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0821 02:46:28.710066    2548 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    error: You must be logged in to the server (the server has asked for the client to provide credentials)\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:347]: Timed out after 300.001s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:332 with:\nkubectl should execute successfully from authorized VM with OIDC token\nUnexpected error:\n    \u003c*errors.errorString | 0xc001943670\u003e: \n    E0820 10:14:05.864231    2546 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0820 10:14:05.892161    2546 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0820 10:14:05.910498    2546 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0820 10:14:05.930980    2546 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0820 10:14:05.946544    2546 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    error: You must be logged in to the server (the server has asked for the client to provide credentials)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:347]: Timed out after 300.001s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:332 with:\nkubectl should execute successfully from authorized VM with OIDC token\nUnexpected error:\n    \u003c*errors.errorString | 0xc001943670\u003e: \n    E0820 10:14:05.864231    2546 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0820 10:14:05.892161    2546 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0820 10:14:05.910498    2546 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0820 10:14:05.930980    2546 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0820 10:14:05.946544    2546 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    error: You must be logged in to the server (the server has asked for the client to provide credentials)\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:347]: Timed out after 300.001s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:332 with:\nkubectl should execute successfully from authorized VM with OIDC token\nUnexpected error:\n    \u003c*errors.errorString | 0xc000f6cc00\u003e: \n    E0818 16:01:10.950779    2564 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0818 16:01:10.970180    2564 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0818 16:01:10.999230    2564 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0818 16:01:11.023177    2564 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0818 16:01:11.041406    2564 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    error: You must be logged in to the server (the server has asked for the client to provide credentials)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:347]: Timed out after 300.001s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:332 with:\nkubectl should execute successfully from authorized VM with OIDC token\nUnexpected error:\n    \u003c*errors.errorString | 0xc000f6cc00\u003e: \n    E0818 16:01:10.950779    2564 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0818 16:01:10.970180    2564 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0818 16:01:10.999230    2564 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0818 16:01:11.023177    2564 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0818 16:01:11.041406    2564 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    error: You must be logged in to the server (the server has asked for the client to provide credentials)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090612430223708160","occurred_at":"2026-08-21T01:30:11Z","signature_id":"f1894cbb696f520fee9ff4522a3cde239d81bc377f24ec4767de138d2980be5a","pr_number":6612,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"6eb333897f7f5d34b2448148e7185a4034de5809364747874cf76879c5e07670","pr_number":6583,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089723779814002688","occurred_at":"2026-08-18T14:39:04Z","signature_id":"13a65dffc82d494fe8412b7fc78b45c5a755711db7f565e3e3f369bfb464e9de","pr_number":6157,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"db70381daed7f2c0ae7388805235b1692e0241b6bbf9c9fad5adb0318ac4ed06","failure_pattern":"failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"\u003curl\u003e","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.2","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.5110732538330494,"anchor_occurrences":3,"after_last_push_count":1,"trend_counts":[0,0,0,3,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-20T15:48:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.2","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.131.82.17:38030-\u003e172.30.0.10:53: i/o timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.131.82.17:38030-\u003e172.30.0.10:53: i/o timeout","fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.130.38.75:43729-\u003e172.30.0.10:53: i/o timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.130.38.75:43729-\u003e172.30.0.10:53: i/o timeout","fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.130.38.9:59641-\u003e172.30.0.10:53: i/o timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:143]: failed to get latest install version for candidate channel: failed getting controlPlaneVersion: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.130.38.9:59641-\u003e172.30.0.10:53: i/o timeout"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090465920358027264","occurred_at":"2026-08-20T15:48:08Z","signature_id":"f293df76335ff7a793270b509afdcb832220e5fc494a428bb2d05bcb86a7eacd","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090283103464460288","occurred_at":"2026-08-20T03:41:42Z","signature_id":"036ccb61732bbdd95d1fe66f7d3a178bde3724585269d7d722d62ee6e4611402","pr_number":6599,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090248642563674112","occurred_at":"2026-08-20T01:24:45Z","signature_id":"b55dd7b53b01cb139b15a9a7c10b0647e7752eca515812e2b7f5ba7dc4f7c030","pr_number":6615,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":0.5110732538330494,"anchor_occurrences":3,"after_last_push_count":1,"trend_counts":[0,0,2,0,0,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-17"],"prior_runs_affected":15,"prior_last_seen_at":"2026-08-19T16:39:13Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1}],"full_error_samples":["time=2026-08-24T09:44:43.919Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T09:44:44.928Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-08-24T09:44:46.104Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=f40d7b7ab0b2d9646dc8b78fbaa252859442c0d4d4549c8765d66346f06ac123 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4585344-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff40d7b7ab0b2d9646dc8b78fbaa252859442c0d4d4549c8765d66346f06ac123\ntime=2026-08-24T10:14:43.424Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","time=2026-08-19T16:52:19.566Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-19T16:52:21.760Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-19T16:52:23.532Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=b29ce57d48a551877ae83bcdc41f601e4ab16f4893b46fbfe80ac0981dd4dfea portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9953536-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fb29ce57d48a551877ae83bcdc41f601e4ab16f4893b46fbfe80ac0981dd4dfea\ntime=2026-08-19T17:22:19.278Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","time=2026-08-19T12:23:07.894Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-19T12:23:08.895Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-08-19T12:23:10.111Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=4ab82aee364a2c4f78d1f63c483aaade6c8eb3341b871d39ccb8291832806f35 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4234112-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F4ab82aee364a2c4f78d1f63c483aaade6c8eb3341b871d39ccb8291832806f35\ntime=2026-08-19T12:53:07.512Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6655/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091810349014585344","occurred_at":"2026-08-24T08:50:25Z","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","pr_number":6655,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090116384229953536","occurred_at":"2026-08-19T16:39:13Z","signature_id":"546713f253dbb07bdf6d6b240d468c73b1927023c4bbf21dee04e87aaa304e9a","pr_number":6601,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090044954054234112","occurred_at":"2026-08-19T11:55:22Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6547,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"91b64d02be224daeb152d864ec9bb135842f4f0c70f108c5bb00e21886c7eb97","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.34071550255536626,"anchor_occurrences":3,"after_last_push_count":1,"trend_counts":[0,0,0,0,3,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-21T16:22:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:299]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014739f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-54pbsl\" in resourcegroup=\"rg-hs-pre--hgwq5fppp7qz\" to finish deleting: GET https://rp.j6534144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03101c36-9e92-40e9-8333-009c7244042d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03101c36-9e92-40e9-8333-009c7244042d\",\n      \"name\": \"03101c36-9e92-40e9-8333-009c7244042d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T17:20:28.711507975Z\",\n      \"endTime\": \"2026-08-21T17:44:35.276549472Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:299]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014739f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-54pbsl\" in resourcegroup=\"rg-hs-pre--hgwq5fppp7qz\" to finish deleting: GET https://rp.j6534144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03101c36-9e92-40e9-8333-009c7244042d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03101c36-9e92-40e9-8333-009c7244042d\",\n      \"name\": \"03101c36-9e92-40e9-8333-009c7244042d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T17:20:28.711507975Z\",\n      \"endTime\": \"2026-08-21T17:44:35.276549472Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:299]: Unexpected error:\n    \u003c*errors.joinError | 0xc00136c168\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7786mg2bm9fv\" to finish deleting: GET https://rp.j6534144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b466b32a-5f1d-4a5f-ba54-1dab632359ab\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b466b32a-5f1d-4a5f-ba54-1dab632359ab\",\n      \"name\": \"b466b32a-5f1d-4a5f-ba54-1dab632359ab\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T17:20:32.530035089Z\",\n      \"endTime\": \"2026-08-21T17:44:35.27735534Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:299]: Unexpected error:\n    \u003c*errors.joinError | 0xc00136c168\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7786mg2bm9fv\" to finish deleting: GET https://rp.j6534144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b466b32a-5f1d-4a5f-ba54-1dab632359ab\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b466b32a-5f1d-4a5f-ba54-1dab632359ab\",\n      \"name\": \"b466b32a-5f1d-4a5f-ba54-1dab632359ab\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T17:20:32.530035089Z\",\n      \"endTime\": \"2026-08-21T17:44:35.27735534Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012240a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-5xdbvmxgwd5n\" to finish deleting: GET https://rp.j7397632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a782820-358c-4562-a28f-3ad7c668ca68\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a782820-358c-4562-a28f-3ad7c668ca68\",\n      \"name\": \"8a782820-358c-4562-a28f-3ad7c668ca68\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T04:23:24.977266828Z\",\n      \"endTime\": \"2026-08-21T04:47:26.492488581Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012240a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-5xdbvmxgwd5n\" to finish deleting: GET https://rp.j7397632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a782820-358c-4562-a28f-3ad7c668ca68\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8a782820-358c-4562-a28f-3ad7c668ca68\",\n      \"name\": \"8a782820-358c-4562-a28f-3ad7c668ca68\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T04:23:24.977266828Z\",\n      \"endTime\": \"2026-08-21T04:47:26.492488581Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6647/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090836865526534144","occurred_at":"2026-08-21T16:22:00Z","signature_id":"0d1b73ed050b18456353cb4d2352218fe21c85c9eda609d938d0ed0a6d1dc131","pr_number":6647,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6647/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090836865526534144","occurred_at":"2026-08-21T16:22:00Z","signature_id":"67ec7c174fce546a47f303fa82220ae45719d138d9e2e4b62f431576840c8059","pr_number":6647,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090644128227397632","occurred_at":"2026-08-21T03:36:17Z","signature_id":"3db2ec536258c92a9174667711d55e05fd5d9a3ef77f875aa5ea0530b4cb8a7b","pr_number":6638,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"604364d97bb0ab0255e5c33341b7e0841ff36bd1d0c70f547552741d3f27bbc9","failure_pattern":"Unable to connect to the server: EOF)","search_query":"Unable to connect to the server: EOF)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,3,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T15:06:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS and private ingress and verify both are only reachable from VNet","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS, private ingress, and private KeyVault and verify all are operational","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:152]: KAS should be reachable from VM via private endpoint (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nUnable to connect to the server: EOF)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:152]: KAS should be reachable from VM via private endpoint (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nUnable to connect to the server: EOF)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:196]: kubectl version should show Server Version, proving KAS is reachable from VM (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nUnable to connect to the server: EOF)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:196]: kubectl version should show Server Version, proving KAS is reachable from VM (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nUnable to connect to the server: EOF)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:165]: kubectl version should show Server Version, proving KAS is reachable from VM (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nUnable to connect to the server: EOF)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:165]: kubectl version should show Server Version, proving KAS is reachable from VM (output: Client Version: v1.36.4\nKustomize Version: v5.8.1\nUnable to connect to the server: EOF)\nExpected\n    \u003cstring\u003e: ...\nto contain substring\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090817798367350784","occurred_at":"2026-08-21T15:06:21Z","signature_id":"12740fe2d6d61cf441e4366b28f12d088ca342286578edc02db60262d6b170c1","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090817798367350784","occurred_at":"2026-08-21T15:06:21Z","signature_id":"9e38ec5c20d23e8d209c2c2b4f32636885e8207d2981ec8540bd0b7c2a8e09e6","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090817798367350784","occurred_at":"2026-08-21T15:06:21Z","signature_id":"9e38ec5c20d23e8d209c2c2b4f32636885e8207d2981ec8540bd0b7c2a8e09e6","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster fully-private, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20260630FromParam for cluster fully-private in resource group fully-private-grwst6zxhql9, error: failed waiting for cluster=\"fully-private\" in resourcegroup=\"fully-private-grwst6zxhql9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20260630FromParam for cluster fully-private in resource group fully-private-grwst6zxhql9, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":3,"after_last_push_count":0,"also_in":["PROD","STG"],"trend_counts":[0,0,0,3,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":35,"prior_last_seen_at":"2026-08-20T14:20:26Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS and private ingress and verify both are only reachable from VNet","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS, private ingress, and private KeyVault and verify all are operational","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:100]: failed to create fully private HCP cluster \"fully-private\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0029fc600\u003e: \n    failed to create HCP cluster fully-private, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20260630FromParam for cluster fully-private in resource group fully-private-grwst6zxhql9, error: failed waiting for cluster=\"fully-private\" in resourcegroup=\"fully-private-grwst6zxhql9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20260630FromParam for cluster fully-private in resource group fully-private-grwst6zxhql9, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:100]: failed to create fully private HCP cluster \"fully-private\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0029fc600\u003e: \n    failed to create HCP cluster fully-private, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20260630FromParam for cluster fully-private in resource group fully-private-grwst6zxhql9, error: failed waiting for cluster=\"fully-private\" in resourcegroup=\"fully-private-grwst6zxhql9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20260630FromParam for cluster fully-private in resource group fully-private-grwst6zxhql9, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:91]: failed to create private HCP cluster \"priv-np-scale\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002136a50\u003e: \n    failed to create HCP cluster priv-np-scale, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster priv-np-scale in resource group priv-np-scale-zkfmfd2hhlvt, error: failed waiting for cluster=\"priv-np-scale\" in resourcegroup=\"priv-np-scale-zkfmfd2hhlvt\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster priv-np-scale in resource group priv-np-scale-zkfmfd2hhlvt, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:91]: failed to create private HCP cluster \"priv-np-scale\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002136a50\u003e: \n    failed to create HCP cluster priv-np-scale, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster priv-np-scale in resource group priv-np-scale-zkfmfd2hhlvt, error: failed waiting for cluster=\"priv-np-scale\" in resourcegroup=\"priv-np-scale-zkfmfd2hhlvt\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster priv-np-scale in resource group priv-np-scale-zkfmfd2hhlvt, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:116]: failed to create fully private HCP cluster \"full-priv-kv\" with private KV\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001614e70\u003e: \n    failed waiting for cluster=\"full-priv-kv\" in resourcegroup=\"full-priv-kv-h8k8sz27jdqk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20260630AndWait for cluster full-priv-kv in resource group full-priv-kv-h8k8sz27jdqk, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:116]: failed to create fully private HCP cluster \"full-priv-kv\" with private KV\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001614e70\u003e: \n    failed waiting for cluster=\"full-priv-kv\" in resourcegroup=\"full-priv-kv-h8k8sz27jdqk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20260630AndWait for cluster full-priv-kv in resource group full-priv-kv-h8k8sz27jdqk, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090443849187987456","occurred_at":"2026-08-20T14:20:26Z","signature_id":"030164acb30f2dbbf672de3a51088927223cf9230bbe93ac2bc5757d097d3729","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090443849187987456","occurred_at":"2026-08-20T14:20:26Z","signature_id":"5e749b5d7218678f3f8683b1e0d2510850d1213f30e492a7e9e0982cd6cdea23","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090443849187987456","occurred_at":"2026-08-20T14:20:26Z","signature_id":"e69e5a85d0e016d4801e1345cf988a8d0b41e00f41b331baea3bdb9477a0c546","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9c46f9d1784a1065f8ede3ae89afa82edefc408d56f3767893d042bb59ac601c","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.34071550255536626,"anchor_occurrences":2,"after_last_push_count":1,"also_in":["PROD","STG"],"trend_counts":[0,0,0,1,0,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":23,"prior_last_seen_at":"2026-08-20T22:52:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091895460682797056","occurred_at":"2026-08-24T14:28:36Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090572687599472640","occurred_at":"2026-08-20T22:52:24Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-s6l5c.apps.aro.v20260901.jnkt.j6034176.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s6l5c.apps.aro.v20260901.jnkt.j6034176.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s6l5c.apps.aro.v20260901.jnkt.j6034176.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.34071550255536626,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,1,0,0,1,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-22T03:47:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-tpmsc2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9ab40\u003e: \n    DNS for route host agnhost-e2e-sample-app-vdv8k.apps.aro.r6l0b2y8t4z4p9n.rppd.j6613120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-vdv8k.apps.aro.r6l0b2y8t4z4p9n.rppd.j6613120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vdv8k.apps.aro.r6l0b2y8t4z4p9n.rppd.j6613120.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:115]: failed to verify simple web app runs on cluster \"cluster-hs-pre-tpmsc2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9ab40\u003e: \n    DNS for route host agnhost-e2e-sample-app-vdv8k.apps.aro.r6l0b2y8t4z4p9n.rppd.j6613120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-vdv8k.apps.aro.r6l0b2y8t4z4p9n.rppd.j6613120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vdv8k.apps.aro.r6l0b2y8t4z4p9n.rppd.j6613120.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:131]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010b23c0\u003e: \n    DNS for route host agnhost-e2e-sample-app-s6l5c.apps.aro.v20260901.jnkt.j6034176.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s6l5c.apps.aro.v20260901.jnkt.j6034176.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s6l5c.apps.aro.v20260901.jnkt.j6034176.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:131]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010b23c0\u003e: \n    DNS for route host agnhost-e2e-sample-app-s6l5c.apps.aro.v20260901.jnkt.j6034176.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s6l5c.apps.aro.v20260901.jnkt.j6034176.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s6l5c.apps.aro.v20260901.jnkt.j6034176.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091009379636613120","occurred_at":"2026-08-22T03:47:39Z","signature_id":"0522283d1fc1a443421be885233a74dcbe6c90c68314e2d70f47677b913d2059","pr_number":6607,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6600/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090054287706034176","occurred_at":"2026-08-19T12:32:25Z","signature_id":"3f9698e822bb6121785dfd4c9a4ed9d11389bae85f981bd4dacd92dce39fcb8e","pr_number":6600,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"2cc2a39959875f671b6207a8cb1637964bc140701e12d99f249891f56ec6e90a","failure_pattern":"ERROR CODE: DeploymentActive","search_query":"ERROR CODE: DeploymentActive","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/management/output (stamp=1)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.34071550255536626,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[1,0,1,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-20T11:31:49Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Fleet.Registration/management/output (stamp=1)","occurrences":1},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/management/output (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-08-20T12:10:57.311Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=output stamp=1 description=\"Step output\\n  Kind: ARM\\n  Template: ./../dev-infrastructure/templates/output-mgmt.bicep\\n  Parameters: ./../dev-infrastructure/configurations/output-mgmt.tmpl.bicepparam\"\ntime=2026-08-20T12:10:57.943Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=output stamp=1\ntime=2026-08-20T12:10:58.421Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=management step=output stamp=1 err=\"stamp 1: failed to run ARM step: failed to create deployment: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5367552-mgmt-1/providers/Microsoft.Resources/deployments/b33ca9cd40b2fdfdad4fc961f59230c1353c681efeada5d41f51b14954fa4cd6\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: DeploymentActive\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentActive\\\",\\n    \\\"message\\\": \\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5367552-mgmt-1/providers/Microsoft.Resources/deployments/b33ca9cd40b2fdfdad4fc961f59230c1353c681efeada5d41f51b14954fa4cd6' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '8/20/2026 12:10:57 PM' with correlationId '9748f509-1eae-448e-a55b-8ae0259f31d6', and will expire at '8/27/2026 12:10:57 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-08-19T01:54:26.847Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 description=\"Step output\\n  Kind: ARM\\n  Template: ../../dev-infrastructure/templates/output-mgmt.bicep\\n  Parameters: ../../dev-infrastructure/configurations/output-mgmt.tmpl.bicepparam\"\ntime=2026-08-19T01:54:27.623Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1\ntime=2026-08-19T01:54:28.068Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Fleet.Registration resourceGroup=management step=output stamp=1 err=\"stamp 1: failed to run ARM step: failed to create deployment: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9945216-mgmt-1/providers/Microsoft.Resources/deployments/21fa006528782108614538657b128ed438c46927450e49bcd34d09fae1d95b96\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: DeploymentActive\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentActive\\\",\\n    \\\"message\\\": \\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9945216-mgmt-1/providers/Microsoft.Resources/deployments/21fa006528782108614538657b128ed438c46927450e49bcd34d09fae1d95b96' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '8/19/2026 1:54:27 AM' with correlationId 'b1f79be8-ea5e-4ca3-9b19-2374cbb23eb8', and will expire at '8/26/2026 1:54:27 AM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090401415305367552","occurred_at":"2026-08-20T11:31:49Z","signature_id":"60f61d5cb7c890572bc4c320ced0f64c99e01d9ec051384d6144b65fd7b48f8a","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089884673239945216","occurred_at":"2026-08-19T01:18:28Z","signature_id":"dcf733b79532628fb81ebf58d69836a0011b2d8445bc977f5d9198084d7b8eca","pr_number":6588,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"c3db9c7d221b5f14f4cb052b432def48559d4c9a3619db1db26cc57a99142ed9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.34071550255536626,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-21T04:39:49Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--wkl8qfr9l4cs/cluster-hs-pre-jtp7jx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009c0960\u003e: \n    failed to create HCP cluster cluster-hs-pre-jtp7jx: failed waiting for cluster=\"cluster-hs-pre-jtp7jx\" in resourcegroup=\"rg-hs-pre--wkl8qfr9l4cs\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df253c01-fd54-4465-9725-47d0942d268e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df253c01-fd54-4465-9725-47d0942d268e\",\n      \"name\": \"df253c01-fd54-4465-9725-47d0942d268e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:18:48.695053949Z\",\n      \"endTime\": \"2026-08-24T15:37:57.993163664Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--wkl8qfr9l4cs/cluster-hs-pre-jtp7jx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009c0960\u003e: \n    failed to create HCP cluster cluster-hs-pre-jtp7jx: failed waiting for cluster=\"cluster-hs-pre-jtp7jx\" in resourcegroup=\"rg-hs-pre--wkl8qfr9l4cs\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df253c01-fd54-4465-9725-47d0942d268e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df253c01-fd54-4465-9725-47d0942d268e\",\n      \"name\": \"df253c01-fd54-4465-9725-47d0942d268e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:18:48.695053949Z\",\n      \"endTime\": \"2026-08-24T15:37:57.993163664Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f3eb00\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-lt8bjkrkn9th\" to finish creating: GET https://rp.j7796352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b8c2ca2-6195-40b4-ba47-51005f5efa92\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b8c2ca2-6195-40b4-ba47-51005f5efa92\",\n      \"name\": \"0b8c2ca2-6195-40b4-ba47-51005f5efa92\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T05:27:49.649301915Z\",\n      \"endTime\": \"2026-08-21T05:46:58.402970178Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f3eb00\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-lt8bjkrkn9th\" to finish creating: GET https://rp.j7796352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b8c2ca2-6195-40b4-ba47-51005f5efa92\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0b8c2ca2-6195-40b4-ba47-51005f5efa92\",\n      \"name\": \"0b8c2ca2-6195-40b4-ba47-51005f5efa92\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T05:27:49.649301915Z\",\n      \"endTime\": \"2026-08-21T05:46:58.402970178Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"113a28e0e41ddb16e2d1b2e27f386f97c1c16af7bfd0bd1391cd74394fd64a44","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090660118717796352","occurred_at":"2026-08-21T04:39:49Z","signature_id":"c41bef6c51b50ccc02005212254243b6d9e7442f62932fa4787fc2a446e87426","pr_number":6584,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1e1c28b43b9490600a25203b94e222bbccfa5a8eac1aa50a00348fd1b2cc72d0","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.34071550255536626,"anchor_occurrences":2,"after_last_push_count":0,"also_in":["STG"],"trend_counts":[0,0,0,0,0,1,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-22T18:55:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a808e0\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-cgt6b2gspgmg\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c64a33f9-1d34-4ca6-89af-ad50a67745b6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c64a33f9-1d34-4ca6-89af-ad50a67745b6\",\n      \"name\": \"c64a33f9-1d34-4ca6-89af-ad50a67745b6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:19:10.873312133Z\",\n      \"endTime\": \"2026-08-24T15:38:17.935941947Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-node-tuning-operator, redhat-marketplace-catalog, cluster-network-operator, redhat-operators-catalog, catalog-operator, openshift-route-controller-manager, ingress-operator, hosted-cluster-config-operator, packageserver, cluster-policy-controller, cluster-storage-operator, certified-operators-catalog, olm-operator, openshift-controller-manager, csi-snapshot-controller-operator, community-operators-catalog, dns-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a808e0\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-cgt6b2gspgmg\" to finish creating: GET https://rp.j1362816.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c64a33f9-1d34-4ca6-89af-ad50a67745b6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c64a33f9-1d34-4ca6-89af-ad50a67745b6\",\n      \"name\": \"c64a33f9-1d34-4ca6-89af-ad50a67745b6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:19:10.873312133Z\",\n      \"endTime\": \"2026-08-24T15:38:17.935941947Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, cluster-node-tuning-operator, redhat-marketplace-catalog, cluster-network-operator, redhat-operators-catalog, catalog-operator, openshift-route-controller-manager, ingress-operator, hosted-cluster-config-operator, packageserver, cluster-policy-controller, cluster-storage-operator, certified-operators-catalog, olm-operator, openshift-controller-manager, csi-snapshot-controller-operator, community-operators-catalog, dns-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-mc8glc\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ef0ae0\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-mc8glc: failed waiting for cluster=\"cluster-zstream-5-0-mc8glc\" in resourcegroup=\"rg-zstream-upgrade-5-0-2vkcfjjhrfq7\" to finish creating: GET https://rp.j1190016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5fff6b19-6c32-4c6e-be68-2f235482a9e4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5fff6b19-6c32-4c6e-be68-2f235482a9e4\",\n      \"name\": \"5fff6b19-6c32-4c6e-be68-2f235482a9e4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T19:50:28.400198816Z\",\n      \"endTime\": \"2026-08-22T20:09:32.092034142Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, ingress-operator, cluster-image-registry-operator, certified-operators-catalog, cluster-storage-operator, dns-operator, cluster-network-operator, cluster-node-tuning-operator, catalog-operator, redhat-marketplace-catalog, cluster-version-operator, openshift-controller-manager, cluster-policy-controller, csi-snapshot-controller-operator, olm-operator, openshift-route-controller-manager, packageserver, community-operators-catalog, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-mc8glc\" with version 5.0.0-ec.5 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ef0ae0\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-mc8glc: failed waiting for cluster=\"cluster-zstream-5-0-mc8glc\" in resourcegroup=\"rg-zstream-upgrade-5-0-2vkcfjjhrfq7\" to finish creating: GET https://rp.j1190016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5fff6b19-6c32-4c6e-be68-2f235482a9e4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5fff6b19-6c32-4c6e-be68-2f235482a9e4\",\n      \"name\": \"5fff6b19-6c32-4c6e-be68-2f235482a9e4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T19:50:28.400198816Z\",\n      \"endTime\": \"2026-08-22T20:09:32.092034142Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator, ingress-operator, cluster-image-registry-operator, certified-operators-catalog, cluster-storage-operator, dns-operator, cluster-network-operator, cluster-node-tuning-operator, catalog-operator, redhat-marketplace-catalog, cluster-version-operator, openshift-controller-manager, cluster-policy-controller, csi-snapshot-controller-operator, olm-operator, openshift-route-controller-manager, packageserver, community-operators-catalog, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091896822531362816","occurred_at":"2026-08-24T14:34:02Z","signature_id":"6b56df09ac5297e89b7853c7486e42c460930ae4b037271fddfef92b6ce0846a","pr_number":6575,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091237883951190016","occurred_at":"2026-08-22T18:55:32Z","signature_id":"d9db2ecb155b483c3a1b51ce5eaa636dea02ee486d2e3c06c7f286e33f5c9a7c","pr_number":6557,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a336a16a5821e0f9bdc957e0ad25795b90745b377edb258591a36a05e1d4d901","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.34071550255536626,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[1,0,1,0,0,1,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-22T00:25:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f624a0\u003e: \n    failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-b4jk7qnglhxf\" to finish creating: GET https://rp.j3592192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/034c0819-3152-42a0-b8e3-f7eba3b93869\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/034c0819-3152-42a0-b8e3-f7eba3b93869\",\n      \"name\": \"034c0819-3152-42a0-b8e3-f7eba3b93869\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:19:06.679031657Z\",\n      \"endTime\": \"2026-08-22T01:38:11.645385258Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:101]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f624a0\u003e: \n    failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-b4jk7qnglhxf\" to finish creating: GET https://rp.j3592192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/034c0819-3152-42a0-b8e3-f7eba3b93869\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/034c0819-3152-42a0-b8e3-f7eba3b93869\",\n      \"name\": \"034c0819-3152-42a0-b8e3-f7eba3b93869\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:19:06.679031657Z\",\n      \"endTime\": \"2026-08-22T01:38:11.645385258Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-rtnpcj\" with version 4.21.28 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000900180\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-rtnpcj: failed waiting for cluster=\"cluster-zstream-4-21-rtnpcj\" in resourcegroup=\"rg-zstream-upgrade-4-21-x7f8mnkj2vz8\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a8aff4d-f2fe-451d-ac5c-e72fcc4613ae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a8aff4d-f2fe-451d-ac5c-e72fcc4613ae\",\n      \"name\": \"4a8aff4d-f2fe-451d-ac5c-e72fcc4613ae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:28:49.971564127Z\",\n      \"endTime\": \"2026-08-19T19:47:49.803372891Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-rtnpcj\" with version 4.21.28 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000900180\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-rtnpcj: failed waiting for cluster=\"cluster-zstream-4-21-rtnpcj\" in resourcegroup=\"rg-zstream-upgrade-4-21-x7f8mnkj2vz8\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a8aff4d-f2fe-451d-ac5c-e72fcc4613ae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a8aff4d-f2fe-451d-ac5c-e72fcc4613ae\",\n      \"name\": \"4a8aff4d-f2fe-451d-ac5c-e72fcc4613ae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:28:49.971564127Z\",\n      \"endTime\": \"2026-08-19T19:47:49.803372891Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090958470403592192","occurred_at":"2026-08-22T00:25:22Z","signature_id":"a58d3dae210b0ca58aefc8f28c941d375561d97193e90e960861be497d6215a9","pr_number":6554,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"bef5be2331817cad7a640f5b07ec3c12d25ea81d5e5773389458ac86e4d299d5","pr_number":6599,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"2c8884cfc8ca38081890b0e85c74fa47ff1c59ad6753d36c6398a702d4e61b9d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.34071550255536626,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,1,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-22T00:26:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010ec220\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-wgscgllb56rj\" to finish creating: GET https://rp.j7976064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/924dc541-9da5-4758-82ab-8fceb926a406\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/924dc541-9da5-4758-82ab-8fceb926a406\",\n      \"name\": \"924dc541-9da5-4758-82ab-8fceb926a406\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:17:11.361872214Z\",\n      \"endTime\": \"2026-08-22T01:36:20.474252514Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010ec220\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-wgscgllb56rj\" to finish creating: GET https://rp.j7976064.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/924dc541-9da5-4758-82ab-8fceb926a406\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/924dc541-9da5-4758-82ab-8fceb926a406\",\n      \"name\": \"924dc541-9da5-4758-82ab-8fceb926a406\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:17:11.361872214Z\",\n      \"endTime\": \"2026-08-22T01:36:20.474252514Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005c4a00\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-xmgxlx2qvkgf\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/18aad82a-dc15-4cc0-bfaa-dde9546ca3b4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/18aad82a-dc15-4cc0-bfaa-dde9546ca3b4\",\n      \"name\": \"18aad82a-dc15-4cc0-bfaa-dde9546ca3b4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:28:30.596156848Z\",\n      \"endTime\": \"2026-08-19T19:47:39.777219011Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005c4a00\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-xmgxlx2qvkgf\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/18aad82a-dc15-4cc0-bfaa-dde9546ca3b4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/18aad82a-dc15-4cc0-bfaa-dde9546ca3b4\",\n      \"name\": \"18aad82a-dc15-4cc0-bfaa-dde9546ca3b4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:28:30.596156848Z\",\n      \"endTime\": \"2026-08-19T19:47:39.777219011Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090958673797976064","occurred_at":"2026-08-22T00:26:10Z","signature_id":"2b6787ba53692025280cc868873b273460045f3589a2601b36a19ada55076ff1","pr_number":6557,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"8b391d25577bbef6f7a5af7392059157b117715fdf88ddadc6f7148e824af69c","pr_number":6599,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"02722dbba27bb6f11330d4f12607b8b70d9877ef02a2097d3ac72980c6a37ce3","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.34071550255536626,"anchor_occurrences":2,"after_last_push_count":3,"also_in":["PROD"],"trend_counts":[0,1,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":9,"prior_last_seen_at":"2026-08-21T06:13:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-cnl87q\" with version 4.22.9 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a08460\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-cnl87q: failed waiting for cluster=\"cluster-zstream-4-22-cnl87q\" in resourcegroup=\"rg-zstream-upgrade-4-22-662hvhb82n8n\" to finish creating: GET https://rp.j6488960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ce5f4c84-dc0a-46e6-a0ed-312916e8fa6e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ce5f4c84-dc0a-46e6-a0ed-312916e8fa6e\",\n      \"name\": \"ce5f4c84-dc0a-46e6-a0ed-312916e8fa6e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T07:01:16.242396742Z\",\n      \"endTime\": \"2026-08-21T07:20:20.789740991Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-cnl87q\" with version 4.22.9 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a08460\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-cnl87q: failed waiting for cluster=\"cluster-zstream-4-22-cnl87q\" in resourcegroup=\"rg-zstream-upgrade-4-22-662hvhb82n8n\" to finish creating: GET https://rp.j6488960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ce5f4c84-dc0a-46e6-a0ed-312916e8fa6e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ce5f4c84-dc0a-46e6-a0ed-312916e8fa6e\",\n      \"name\": \"ce5f4c84-dc0a-46e6-a0ed-312916e8fa6e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T07:01:16.242396742Z\",\n      \"endTime\": \"2026-08-21T07:20:20.789740991Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e58240\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-rt9fczslhsfd\" to finish creating: GET https://rp.j7122944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c396cbc9-6606-49f9-91e4-6b21abbce6fd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c396cbc9-6606-49f9-91e4-6b21abbce6fd\",\n      \"name\": \"c396cbc9-6606-49f9-91e4-6b21abbce6fd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T16:30:41.092897918Z\",\n      \"endTime\": \"2026-08-18T16:49:50.005176952Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e58240\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-rt9fczslhsfd\" to finish creating: GET https://rp.j7122944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c396cbc9-6606-49f9-91e4-6b21abbce6fd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c396cbc9-6606-49f9-91e4-6b21abbce6fd\",\n      \"name\": \"c396cbc9-6606-49f9-91e4-6b21abbce6fd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T16:30:41.092897918Z\",\n      \"endTime\": \"2026-08-18T16:49:50.005176952Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090683781416488960","occurred_at":"2026-08-21T06:13:51Z","signature_id":"4ca248892099e2c2d4e4a976050bdfef3c0ae82bdef9070cafe3f93ce794316c","pr_number":6519,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4566/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089740517767122944","occurred_at":"2026-08-18T15:45:39Z","signature_id":"c6a9f42fb2831adef45ef1ded8e8b6500b858b024aad76a8c6c50880c3072eff","pr_number":4566,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fe6b01560ac49646fd0a46c48f672cf2fc0ebbf844ea4576bb0cab7a46df7ce8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.34071550255536626,"anchor_occurrences":2,"after_last_push_count":0,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,1,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-10","2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-22T10:03:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-cbhfzqtvb9th/cluster-candidate-4-21-5bbfxw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea2680\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-5bbfxw: failed waiting for cluster=\"cluster-candidate-4-21-5bbfxw\" in resourcegroup=\"rg-candidate-4-21-cbhfzqtvb9th\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86d82f7-93d6-43a5-99ee-59e2969fd318\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86d82f7-93d6-43a5-99ee-59e2969fd318\",\n      \"name\": \"a86d82f7-93d6-43a5-99ee-59e2969fd318\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T17:28:32.844651935Z\",\n      \"endTime\": \"2026-08-24T17:47:34.560561952Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-cbhfzqtvb9th/cluster-candidate-4-21-5bbfxw should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea2680\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-5bbfxw: failed waiting for cluster=\"cluster-candidate-4-21-5bbfxw\" in resourcegroup=\"rg-candidate-4-21-cbhfzqtvb9th\" to finish creating: GET https://rp.j2463616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86d82f7-93d6-43a5-99ee-59e2969fd318\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a86d82f7-93d6-43a5-99ee-59e2969fd318\",\n      \"name\": \"a86d82f7-93d6-43a5-99ee-59e2969fd318\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T17:28:32.844651935Z\",\n      \"endTime\": \"2026-08-24T17:47:34.560561952Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00099a320\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-vz6d75d4vrkm\" to finish creating: GET https://rp.j3428992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07b48c49-439f-4161-825c-a3f7e1694990\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07b48c49-439f-4161-825c-a3f7e1694990\",\n      \"name\": \"07b48c49-439f-4161-825c-a3f7e1694990\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T10:45:13.181167652Z\",\n      \"endTime\": \"2026-08-22T11:04:18.393945445Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00099a320\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-vz6d75d4vrkm\" to finish creating: GET https://rp.j3428992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07b48c49-439f-4161-825c-a3f7e1694990\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/07b48c49-439f-4161-825c-a3f7e1694990\",\n      \"name\": \"07b48c49-439f-4161-825c-a3f7e1694990\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T10:45:13.181167652Z\",\n      \"endTime\": \"2026-08-22T11:04:18.393945445Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"94e14a4948dabf46772c6e049c6c316329369f07279d585a70084b3a5b07d8b8","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091103857353428992","occurred_at":"2026-08-22T10:03:05Z","signature_id":"c74ff041689685f9d2893c509b1ffbe4205a6aa02762660611f9c1e7f98334d4","pr_number":6651,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"3663019c95fe16313f2b4252470b99823cc8609903410909837b48e27b5716a2","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.22\": no upgrade path to update channel \"\u003cocp-channel\u003e\" is currently available for this cluster; a channel appears in the cluster's desired version channels only when an upgrade edge to a release in that channel exists; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.34071550255536626,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,2,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-21T10:34:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:185]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-jxhbpr\" to 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000827a10\u003e: \n    PATCH https://rp.j1906688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-nk4xz5qj6lkf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-jxhbpr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": no upgrade path to update channel \\\"candidate-4.22\\\" is currently available for this cluster; a channel appears in the cluster's desired version channels only when an upgrade edge to a release in that channel exists\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:185]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-jxhbpr\" to 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000827a10\u003e: \n    PATCH https://rp.j1906688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-nk4xz5qj6lkf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-jxhbpr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": no upgrade path to update channel \\\"candidate-4.22\\\" is currently available for this cluster; a channel appears in the cluster's desired version channels only when an upgrade edge to a release in that channel exists\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:185]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-gq8xsp\" to 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008705a0\u003e: \n    PATCH https://rp.j4357760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-djzz4pj5vnt8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-gq8xsp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": no upgrade path to update channel \\\"candidate-4.22\\\" is currently available for this cluster; a channel appears in the cluster's desired version channels only when an upgrade edge to a release in that channel exists\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:185]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-gq8xsp\" to 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008705a0\u003e: \n    PATCH https://rp.j4357760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-djzz4pj5vnt8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-gq8xsp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": no upgrade path to update channel \\\"candidate-4.22\\\" is currently available for this cluster; a channel appears in the cluster's desired version channels only when an upgrade edge to a release in that channel exists\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6644/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090749265121906688","occurred_at":"2026-08-21T10:34:03Z","signature_id":"3ed03a19f79f0b1e2f0a12412d179fee0ef3e85a880c1f848954a744c9dbd32b","pr_number":6644,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6612/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090748107674357760","occurred_at":"2026-08-21T10:27:08Z","signature_id":"dd42386bd958f86f2faaef3c620305cbb99e3c2d8f582189054329bbccddbac1","pr_number":6612,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d6e88ce654a1e44ba45ba96ba70b0d0713c8f4a869b2a72dfffa4d3817a99e4e","failure_pattern":"VerifyFIPSEnabled failed: failed to check FIPS mode on node \u003cnode\u003e: failed to get pod logs: an error on the server (\"unknown\") has prevented the request from succeeding (get pod \u003cpod\u003e)","search_query":"VerifyFIPSEnabled failed: failed to check FIPS mode on node i3f1v3w5b2n5j4l-np-1-ld9tc-cz8dm: failed to get pod logs: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-s5lz7)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.34071550255536626,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":7,"prior_last_seen_at":"2026-08-20T17:21:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:142]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc00163c708\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node i3f1v3w5b2n5j4l-np-1-ld9tc-cz8dm: failed to get pod logs: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-s5lz7)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:142]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc00163c708\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node i3f1v3w5b2n5j4l-np-1-ld9tc-cz8dm: failed to get pod logs: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-s5lz7)\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:142]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc001412c18\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node u9c4k8b0z1p8s9a-np-1-hhdmj-d5z7z: failed to get pod logs: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-8cndt)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:142]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc001412c18\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node u9c4k8b0z1p8s9a-np-1-hhdmj-d5z7z: failed to get pod logs: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-8cndt)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090489518846119936","occurred_at":"2026-08-20T17:21:54Z","signature_id":"676940ccfda45dd73c139c03c60293cf2f01a67e1fa939f78d3538b86b1422a0","pr_number":6278,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090439005190492160","occurred_at":"2026-08-20T14:00:59Z","signature_id":"0ba8df1b3d711fd1d88f28d00cded9c01dc9d64544970f2a555dd4f85fade54f","pr_number":6584,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"17e5892e8c32879055400ab93a946ce79ad6cc684588825e1e1a488695d06f44","failure_pattern":"failed to poll cluster \"\u003ccluster\u003e\" authorized CIDRs update to completion (timeout '10.000000' minutes)","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.34071550255536626,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,1,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-21T21:25:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:407]: failed to poll cluster \"cidr-connectivity-test\" authorized CIDRs update to completion (timeout '10.000000' minutes)\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:407]: failed to poll cluster \"cidr-connectivity-test\" authorized CIDRs update to completion (timeout '10.000000' minutes)\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090913249020088320","occurred_at":"2026-08-21T21:25:40Z","signature_id":"e903e6214fc854d426efd67d556189c90371a5d9fd33b4c1bbc8c31c92089a9a","pr_number":6632,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6519/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090346173553446912","occurred_at":"2026-08-20T07:52:19Z","signature_id":"e903e6214fc854d426efd67d556189c90371a5d9fd33b4c1bbc8c31c92089a9a","pr_number":6519,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0c8f285ebf7a77151b5c5324a77de4ff3bf14831af7586b962d27c5eacb2a8e6","failure_pattern":"failed to roll out Helm release: resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Agent/management/deploy (stamp=2)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.34071550255536626,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,1,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-21T20:12:06Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Agent/management/deploy (stamp=2)","occurrences":2}],"full_error_samples":["time=2026-08-21T20:50:16.519Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-08-21T20:50:17.093Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2 values=\"map[azure:map[clientId:e8f047ac-b81e-4523-abc9-39d8133a1340 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] broker:map[host:arohcp-ci01-maestro-j3460096.westus3-1.ts.eventgrid.azure.net port:8883] consumerName:hcp-underlay-j3460096-mgmt-2 credsKeyVault:map[name:ah-ci01-mg-j3460096-2 secret:hcp-underlay-j3460096-mgmt-2] deployment:map[zoneCount:4] glog_v:2 image:map[digest:sha256:46e44a9a9cfbd4e157110a6dcc88abdfd569b49315b4ad5088eb42fb69b45b60 registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] installAppliedManifestWorkCRD:false replicas:1 serviceAccountName:maestro sideCar:map[image:map[digest:sha256:bf23c3b25d0f0383736ab2380ed90a25cf5d7aae416de00accc27f8ed6c2e42c registry:mcr.microsoft.com repository:azurelinux/base/nginx]]]\"\ntime=2026-08-21T20:50:17.093Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2\ntime=2026-08-21T20:50:17.310Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2 namespace=maestro\ntime=2026-08-21T20:50:17.570Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2 dryRun=false\ntime=2026-08-21T20:50:17.570Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2\ntime=2026-08-21T20:50:17.785Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2\ntime=2026-08-21T20:55:24.253Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2 err=\"resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-08-21T20:55:24.253Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","time=2026-08-18T23:50:16.511Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-08-18T23:50:17.095Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2 values=\"map[azure:map[clientId:3c23b3ff-ca85-412f-8ee1-123417c85a36 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] broker:map[host:arohcp-ci01-maestro-j0124416.westus3-1.ts.eventgrid.azure.net port:8883] consumerName:hcp-underlay-j0124416-mgmt-2 credsKeyVault:map[name:ah-ci01-mg-j0124416-2 secret:hcp-underlay-j0124416-mgmt-2] deployment:map[zoneCount:4] glog_v:2 image:map[digest:sha256:58352f43f77496307f6e57cf489c23fe2f98a1b2458c762b5fea9ea09249a945 registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] installAppliedManifestWorkCRD:false replicas:1 serviceAccountName:maestro sideCar:map[image:map[digest:sha256:bf23c3b25d0f0383736ab2380ed90a25cf5d7aae416de00accc27f8ed6c2e42c registry:mcr.microsoft.com repository:azurelinux/base/nginx]]]\"\ntime=2026-08-18T23:50:17.095Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2\ntime=2026-08-18T23:50:17.375Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2 namespace=maestro\ntime=2026-08-18T23:50:17.611Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2 dryRun=false\ntime=2026-08-18T23:50:17.611Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2\ntime=2026-08-18T23:50:17.735Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2\ntime=2026-08-18T23:55:23.782Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2 err=\"resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-08-18T23:55:23.782Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090894760083460096","occurred_at":"2026-08-21T20:12:06Z","signature_id":"17bb70335ce31ff41cd691af5d92dd814bd463359d91b7f6775654b4403502e7","pr_number":6631,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6128/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089852994920124416","occurred_at":"2026-08-18T23:12:36Z","signature_id":"bf8de7cbb8732b683cbecab265d0a3bda4185028887c4be0faef0f1243480542","pr_number":6128,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"911f609de0f6334698a9ae03ee21b22f86f10fb03ad2176871cac7bbc4b43a5f","failure_pattern":"failed to select node pool \u003cnodepool\u003e version for \u003cocp-channel\u003e channel: Get \"\u003curl\u003e\": dial tcp: lookup api","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.34071550255536626,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-20T16:56:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.208.156:42093-\u003e172.30.0.10:53: i/o timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.208.156:42093-\u003e172.30.0.10:53: i/o timeout","fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.131.120.38:40027-\u003e172.30.0.10:53: i/o timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.131.120.38:40027-\u003e172.30.0.10:53: i/o timeout"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091939168878858240","occurred_at":"2026-08-24T17:22:18Z","signature_id":"7171e132341694fb10089af1503bd1696a386e558f7ad492e46e056500959d3b","pr_number":5958,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6620/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090483026692673536","occurred_at":"2026-08-20T16:56:05Z","signature_id":"2d5072c48a74af035eca1d7e78f0a0c529eca7b4bf1560e30dd4259ba323f657","pr_number":6620,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f005737f2fefbe69bb0ecb0b0c508c6c810f4416b5487edf7bf43a55898e70a9","failure_pattern":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server]","search_query":"missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server]","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.34071550255536626,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-18T17:40:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:95]: Timed out after 600.003s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000b90f00\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server]\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:95]: Timed out after 600.003s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000b90f00\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server]\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:95]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0013429e0\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server]\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:95]: Timed out after 600.000s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0013429e0\u003e: \n    missing expected log sources: [aro-hcp/aro-hcp-frontend aro-hcp/aro-hcp-backend clusters-service/clusters-service-server]\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6587/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089769452286513152","occurred_at":"2026-08-18T17:40:38Z","signature_id":"99aeed29b4ae494231aa7963eac99d138ba7af879c61d748f09ab75d2948dd25","pr_number":6587,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089574646696382464","occurred_at":"2026-08-18T04:46:32Z","signature_id":"fbe87b57ce8d5e8677b97bf6fc9ecb496862b45f98ec23f5356b77e301364b87","pr_number":6573,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bff39f67862ef72eccdd3c2006d48756f417856635339107ace9fde9b0a679a4","failure_pattern":"stamp 2 scheduling must have requested resources","search_query":"stamp 2 scheduling must have requested resources","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Fleet should have scheduling data for ready management clusters","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":0.34071550255536626,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-22T10:03:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Fleet should have scheduling data for ready management clusters","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:162]: Timed out after 900.001s.\nscheduling data was not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:155 with:\nstamp 2 scheduling must have requested resources\nExpected\n    \u003cv1.ResourceList | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:162]: Timed out after 900.001s.\nscheduling data was not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:155 with:\nstamp 2 scheduling must have requested resources\nExpected\n    \u003cv1.ResourceList | len:0\u003e: ...\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091928139092463616","occurred_at":"2026-08-24T16:34:51Z","signature_id":"b261dd52f676523d8a989fb0761bdfb6871af15b6d8ed758e7d327e30440bdf8","pr_number":6657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091103857353428992","occurred_at":"2026-08-22T10:03:05Z","signature_id":"b261dd52f676523d8a989fb0761bdfb6871af15b6d8ed758e7d327e30440bdf8","pr_number":6651,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ea8fad35cd89a01793d212abe110c72e86c225b2404d2a441b9893a047eca4bb","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00133a200\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-qp6jkgrm7jtk\" to finish creating: GET https://rp.j8136960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d814403-8a1a-482d-afa5-63e68ef4ed09\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d814403-8a1a-482d-afa5-63e68ef4ed09\",\n      \"name\": \"7d814403-8a1a-482d-afa5-63e68ef4ed09\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:13:13.654576687Z\",\n      \"endTime\": \"2026-08-24T21:32:17.358471293Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00133a200\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-qp6jkgrm7jtk\" to finish creating: GET https://rp.j8136960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d814403-8a1a-482d-afa5-63e68ef4ed09\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d814403-8a1a-482d-afa5-63e68ef4ed09\",\n      \"name\": \"7d814403-8a1a-482d-afa5-63e68ef4ed09\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:13:13.654576687Z\",\n      \"endTime\": \"2026-08-24T21:32:17.358471293Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012a4ea0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-26mdm7dwkvm7\" to finish creating: GET https://rp.j8136960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fea58607-6b10-431a-a2a1-cc0c116e10b9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fea58607-6b10-431a-a2a1-cc0c116e10b9\",\n      \"name\": \"fea58607-6b10-431a-a2a1-cc0c116e10b9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:12:09.791388432Z\",\n      \"endTime\": \"2026-08-24T21:31:17.457093893Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012a4ea0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-26mdm7dwkvm7\" to finish creating: GET https://rp.j8136960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fea58607-6b10-431a-a2a1-cc0c116e10b9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fea58607-6b10-431a-a2a1-cc0c116e10b9\",\n      \"name\": \"fea58607-6b10-431a-a2a1-cc0c116e10b9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:12:09.791388432Z\",\n      \"endTime\": \"2026-08-24T21:31:17.457093893Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","signature_id":"ab3b51419f8536317834b49d26ead032c3d2df538686a40306860f09479fc7a4","pr_number":6641,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091984065988136960","occurred_at":"2026-08-24T20:20:43Z","signature_id":"fb242b6b09da99dc7298c2563fdd22c18c2c33f65971895aaf866ce3967b98d3","pr_number":6641,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"34e193d1e971458756c5fedbfac4e10a25751030aa42d3492b58c65f0ceda0fd","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":2,"after_last_push_count":1,"also_in":["PROD","STG"],"trend_counts":[0,0,0,0,0,0,0,2],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-10"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-14T20:39:56Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:304]: failed to create HCP cluster np-noedge-7pzvlw\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e28760\u003e: \n    failed to create HCP cluster np-noedge-7pzvlw: failed waiting for cluster=\"np-noedge-7pzvlw\" in resourcegroup=\"rg-np-noedge-7pzvlw-hmhh49x6v5k9\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\",\n      \"name\": \"d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:29:01.361816726Z\",\n      \"endTime\": \"2026-08-24T21:48:06.132587418Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:304]: failed to create HCP cluster np-noedge-7pzvlw\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e28760\u003e: \n    failed to create HCP cluster np-noedge-7pzvlw: failed waiting for cluster=\"np-noedge-7pzvlw\" in resourcegroup=\"rg-np-noedge-7pzvlw-hmhh49x6v5k9\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\",\n      \"name\": \"d4851c53-78b5-4c3c-8fb8-61c89ed0b4a3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:29:01.361816726Z\",\n      \"endTime\": \"2026-08-24T21:48:06.132587418Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:95]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e95be0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-7w4qb7bx58wj\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c4e3b7e-7834-4bf7-9d07-9139f539790f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c4e3b7e-7834-4bf7-9d07-9139f539790f\",\n      \"name\": \"7c4e3b7e-7834-4bf7-9d07-9139f539790f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:28:50.851778461Z\",\n      \"endTime\": \"2026-08-24T21:47:56.22619205Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:95]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e95be0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-7w4qb7bx58wj\" to finish creating: GET https://rp.j6887040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c4e3b7e-7834-4bf7-9d07-9139f539790f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c4e3b7e-7834-4bf7-9d07-9139f539790f\",\n      \"name\": \"7c4e3b7e-7834-4bf7-9d07-9139f539790f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T21:28:50.851778461Z\",\n      \"endTime\": \"2026-08-24T21:47:56.22619205Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"7467fcd7d827463de8cbe0d7b5f551f437c25ba6ae7fc7de2730705826346049","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091989769276887040","occurred_at":"2026-08-24T20:43:22Z","signature_id":"ca98190d0c65ef7e152d8e004d3f56a8bcff3a1f40939b19e5d08cfc9a992c4b","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":11,"prior_last_seen_at":"2026-08-18T13:32:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:177]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0012d0120\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-rxn7zw namespace are ready: failed to list pods in cilium-connectivity-check-rxn7zw namespace: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:177]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0012d0120\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-rxn7zw namespace are ready: failed to list pods in cilium-connectivity-check-rxn7zw namespace: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001470108\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-6r5pmg namespace are ready: failed to list pods in cilium-connectivity-check-6r5pmg namespace: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001470108\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-6r5pmg namespace are ready: failed to list pods in cilium-connectivity-check-6r5pmg namespace: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6426/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089706907987087360","occurred_at":"2026-08-18T13:32:06Z","signature_id":"285660165cd10da6478cb9e1f13cfde6e457241d2047e02dc2f14c8d3ae3c1ae","pr_number":6426,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6426/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089706907987087360","occurred_at":"2026-08-18T13:32:06Z","signature_id":"c569a5b6ea87090c5f6584c7b1a5fc1f4f2095c27e1e3dcc6c5c076f2352a9fb","pr_number":6426,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c7cc4ecdb77e22f303e5ff2a71d414dfade083e8a3ba8e06dd7fd72186c1004d","failure_pattern":"route host should not be empty","search_query":"route host should not be empty","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS and private ingress and verify both are only reachable from VNet","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,2,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T11:53:35Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS and private ingress and verify both are only reachable from VNet","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS, private ingress, and private KeyVault and verify all are operational","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:209]: Timed out after 305.487s.\nsample app route should become available\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:208 with:\nroute host should not be empty\nExpected\n    \u003cstring\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:209]: Timed out after 305.487s.\nsample app route should become available\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:208 with:\nroute host should not be empty\nExpected\n    \u003cstring\u003e: ...\nnot to be empty","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:236]: Timed out after 304.974s.\nsample app route should become available\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:235 with:\nroute host should not be empty\nExpected\n    \u003cstring\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:236]: Timed out after 304.974s.\nsample app route should become available\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:235 with:\nroute host should not be empty\nExpected\n    \u003cstring\u003e: ...\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090769277882732544","occurred_at":"2026-08-21T11:53:35Z","signature_id":"21340eb6aa41c283e6be7304c4a94150fcd6181a4804c245c5bebf54d04d0a5d","pr_number":6580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090769277882732544","occurred_at":"2026-08-21T11:53:35Z","signature_id":"e6609696a4b8413bbfad821326c2f67aa530f40a06835ff069ea866a2c92cbf1","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5","failure_pattern":"--------------------------------------------------------------------------------","search_query":"--------------------------------------------------------------------------------","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[1,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-19T08:55:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:131]: nodepool creation errors: [failed to create NodePool np-update-nodes: failed waiting for nodepool=\"np-update-nodes\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-v9sgt7z9p57z\" to finish creating: GET https://rp.j8473600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0d392562-a61e-4063-8716-f8bbe26726fb\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0d392562-a61e-4063-8716-f8bbe26726fb\",\n  \"name\": \"0d392562-a61e-4063-8716-f8bbe26726fb\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-08-19T09:49:09.811303693Z\",\n  \"endTime\": \"2026-08-19T10:08:11.65820307Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:1, cap:1\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:131]: nodepool creation errors: [failed to create NodePool np-update-nodes: failed waiting for nodepool=\"np-update-nodes\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-v9sgt7z9p57z\" to finish creating: GET https://rp.j8473600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0d392562-a61e-4063-8716-f8bbe26726fb\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0d392562-a61e-4063-8716-f8bbe26726fb\",\n  \"name\": \"0d392562-a61e-4063-8716-f8bbe26726fb\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-08-19T09:49:09.811303693Z\",\n  \"endTime\": \"2026-08-19T10:08:11.65820307Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:1, cap:1\u003e: ...\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089999791768473600","occurred_at":"2026-08-19T08:55:55Z","signature_id":"2bedfe93590f3320ce1a1d2b5e550c02a75b1d68402e0b650ded60bc95e7e3d8","pr_number":6588,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b55233e1f233919ecc579e6967f76823775cbaf35c5ee37e0a8883b46726c324","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AllocationFailed; detail message The VM allocation failed due to an internal error. Please retry later or try deploying to a different location.; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-10"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-16T12:18:51Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-08-24T19:10:07.135Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-08-24T19:10:08.078Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-08-24T19:10:08.961Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=6b7c37d7d05e023c4460016a815a9ccce8683c2a1a50686369f95e5de1b45b05 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6400512-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6b7c37d7d05e023c4460016a815a9ccce8683c2a1a50686369f95e5de1b45b05\ntime=2026-08-24T19:19:10.348Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.Resources/deployments/6b7c37d7d05e023c4460016a815a9ccce8683c2a1a50686369f95e5de1b45b05/operationStatuses/08584140082768889498\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.Resources/deployments/cluster-3vizzf2ahtjlc\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.Resources/deployments/user-agent-pools\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n                    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j6400512-mgmt-2/agentPools/userswft1\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n                    \\\\\\\"details\\\\\\\": [\\\\r\\\\n                      {\\\\r\\\\n                        \\\\\\\"code\\\\\\\": \\\\\\\"AllocationFailed\\\\\\\",\\\\r\\\\n                        \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6400512-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft1-38184404-vmss'. Please try alternative sizes 'Standard_E16ds_v6' (zones 2, 3, 1), 'Standard_E16ds_v6' (eastus, zones 1, 2, 3), 'Standard_E16ds_v6' (eastus2, zones 2, 1, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation.\\\\\\\",\\\\r\\\\n                        \\\\\\\"details\\\\\\\": [\\\\r\\\\n                          {\\\\r\\\\n                            \\\\\\\"code\\\\\\\": \\\\\\\"AllocationFailed\\\\\\\",\\\\r\\\\n                            \\\\\\\"target\\\\\\\": \\\\\\\"3\\\\\\\",\\\\r\\\\n                            \\\\\\\"message\\\\\\\": \\\\\\\"The VM allocation failed due to an internal error. Please retry later or try deploying to a different location.\\\\\\\"\\\\r\\\\n                          }\\\\r\\\\n                        ]\\\\r\\\\n                      }\\\\r\\\\n                    ]\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091957267166400512","occurred_at":"2026-08-24T18:34:13Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"50b4072821c611f78915fffb00974b54efe31284ac524891f02603d30b54dc67","failure_pattern":"ERROR CODE: DeploymentFailed; detail code BadRequest; detail message The requested operation cannot be performed because the database account arohcpci01-rp-j2394624 is in the process of being created. Please wait until account creation completes and retry the operation.\\\\\\\\\\\\\\ \\\\\\\\\\\\ ActivityId: \u003cuuid\u003e, Microsoft.Azure.Documents.Common/2.14.0\"\\\\\\ \\\\ }; provider Microsoft.DocumentDB","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T20:51:11Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1}],"full_error_samples":["time=2026-08-19T21:09:01.009Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-08-19T21:09:03.163Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-08-19T21:09:04.592Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=5d7d7fd1d17d5cb539e3ba4395d742a56d454dcabf04b8e8cd3c81884c4e29c2 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2394624%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F5d7d7fd1d17d5cb539e3ba4395d742a56d454dcabf04b8e8cd3c81884c4e29c2\ntime=2026-08-19T21:11:05.314Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2394624/providers/Microsoft.Resources/deployments/5d7d7fd1d17d5cb539e3ba4395d742a56d454dcabf04b8e8cd3c81884c4e29c2/operationStatuses/08584144331418179698\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2394624/providers/Microsoft.Resources/deployments/rp-cosmos-account\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2394624/providers/Microsoft.DocumentDB/databaseAccounts/arohcpci01-rp-j2394624\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"BadRequest\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n  \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"The requested operation cannot be performed because the database account arohcpci01-rp-j2394624 is in the process of being created. Please wait until account creation completes and retry the operation.\\\\\\\\\\\\\\\\r\\\\\\\\\\\\\\\\nActivityId: 02191ce0-56b4-4967-b294-a61f839cd58d, Microsoft.Azure.Documents.Common/2.14.0\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090179795802394624","occurred_at":"2026-08-19T20:51:11Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":6492,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cb9a7ba2d967015ac741b1ba2401e95d8184541adbccfc98565936e10e7aa68c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ConcurrentFederatedIdentityCredentialsWritesForSingleManagedIdentity; detail message Too many Federated Identity Credentials are written concurrently for the managed identity '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/microsoft.managedidentity/userassignedidentities/fleet'. Concurrent Federated Identity Credentials writes under the same managed identity are not supported.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T17:25:51Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1}],"full_error_samples":["time=2026-08-21T17:53:50.354Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-08-21T17:53:52.743Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-08-21T17:53:54.610Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=3dbafda6428a192bbb810e342f38cdf4f95c2f9f6f0490ebf2e0d35b2ec09729 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9453824-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F3dbafda6428a192bbb810e342f38cdf4f95c2f9f6f0490ebf2e0d35b2ec09729\ntime=2026-08-21T18:05:27.260Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9453824-svc/providers/Microsoft.Resources/deployments/3dbafda6428a192bbb810e342f38cdf4f95c2f9f6f0490ebf2e0d35b2ec09729/operationStatuses/08584142720518233456\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9453824-svc/providers/Microsoft.Resources/deployments/cluster-bptwqnbhgueys\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ConcurrentFederatedIdentityCredentialsWritesForSingleManagedIdentity\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Too many Federated Identity Credentials are written concurrently for the managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9453824-svc/providers/microsoft.managedidentity/userassignedidentities/fleet'. Concurrent Federated Identity Credentials writes under the same managed identity are not supported.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6650/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090852896869453824","occurred_at":"2026-08-21T17:25:51Z","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","pr_number":6650,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8097f2fb40f9dda7135748f56dd66bc27b12f2b97919326dd7b3b4c74d914621","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ResourceConflict; detail message ETag mismatch on \u003cuuid\u003e; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T05:59:26Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1}],"full_error_samples":["time=2026-08-21T06:22:09.432Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-08-21T06:22:11.097Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-08-21T06:22:13.018Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=e422eaa0da3d95490d2441b44a33daba3f65b59ac89f70d027fcb6e1074a2f11 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2081536%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fe422eaa0da3d95490d2441b44a33daba3f65b59ac89f70d027fcb6e1074a2f11\ntime=2026-08-21T06:28:14.734Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2081536/providers/Microsoft.Resources/deployments/e422eaa0da3d95490d2441b44a33daba3f65b59ac89f70d027fcb6e1074a2f11/operationStatuses/08584143135538485032\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2081536/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2081536/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j2081536\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"ResourceConflict\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"ETag mismatch on 28111326-98a3-4ad2-87d8-5c0102bdf0cd\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090680154262081536","occurred_at":"2026-08-21T05:59:26Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":6640,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bb424b290c173083428a34ac21b9472b0a9be4d1c592b771df2c71e46b269465","failure_pattern":"ERROR CODE: DeploymentFailed; detail code StorageAccountOperationInProgress; detail message An operation is currently performing on this storage account that requires exclusive access.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/infra (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/infra (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-08-24T16:09:49.176Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 description=\"Step infra\\n  Kind: ARM\\n  Template: templates/mgmt-infra.bicep\\n  Parameters: configurations/mgmt-infra.tmpl.bicepparam\"\ntime=2026-08-24T16:09:49.840Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2\ntime=2026-08-24T16:09:50.553Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 deployment=c783e3e3d697ee4d22dffbab4271479e105f154955a91c3a47e9819df57ac290 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1968512-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fc783e3e3d697ee4d22dffbab4271479e105f154955a91c3a47e9819df57ac290\ntime=2026-08-24T16:11:20.899Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1968512-mgmt-2/providers/Microsoft.Resources/deployments/c783e3e3d697ee4d22dffbab4271479e105f154955a91c3a47e9819df57ac290/operationStatuses/08584140190952500816\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1968512-mgmt-2/providers/Microsoft.Resources/deployments/hcp-backups-storage\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1968512-mgmt-2/providers/Microsoft.Resources/deployments/hcpBackupsStorageAccount\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1968512-mgmt-2/providers/Microsoft.Resources/deployments/hcpBackupsStorageAccount\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"StorageAccountOperationInProgress\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"An operation is currently performing on this storage account that requires exclusive access.\\\\\\\"\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6659/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091914307531968512","occurred_at":"2026-08-24T15:43:31Z","signature_id":"d517e51fc5b09c819903bacd247416bcbbe167939922787f3a2438a125b3fdc2","pr_number":6659,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ca6140fe6832e567dbdbd62e69723ab444b3c85ac16836f37268901fc2e842b0","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Timeout; detail message Operation is being Canceled due to timeout.; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1}],"full_error_samples":["time=2026-08-24T07:13:06.981Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-08-24T07:13:09.018Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-08-24T07:13:10.536Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9665ebaf931e927bcf7f137293ddd5a9262983f1f2075a0dff43a22749c6dc59 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1860736%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9665ebaf931e927bcf7f137293ddd5a9262983f1f2075a0dff43a22749c6dc59\ntime=2026-08-24T07:29:44.382Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1860736/providers/Microsoft.Resources/deployments/9665ebaf931e927bcf7f137293ddd5a9262983f1f2075a0dff43a22749c6dc59/operationStatuses/08584140512958847265\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1860736/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1860736/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1860736\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Timeout\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Operation is being Canceled due to timeout.\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091780518591860736","occurred_at":"2026-08-24T06:51:53Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":6278,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"36e7932077fe0de3697c04fb0ba302a7298b56f122f6c49db035b3d764e4980b","failure_pattern":"ERROR CODE: DeploymentStackDeploymentFailed; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentStackDeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nodepools (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nodepools (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-08-24T08:15:25.803Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nodepools stamp=2 description=\"Step nodepools\\n  Kind: ARMStack\\n  Template: templates/aks-nodepools.bicep\\n  Parameters: configurations/mgmt-cluster-nodepools.tmpl.bicepparam\"\ntime=2026-08-24T08:18:16.150Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nodepools stamp=2 err=\"stamp 2: failed to run ARM step: failed to wait for deployment stack at resource group scope: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/67d8fd4b-85de-4d57-a829-7ce0bf268c5d\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentStackDeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/locations/westus2/deploymentStackOperationStatus/67d8fd4b-85de-4d57-a829-7ce0bf268c5d\\\",\\n  \\\"name\\\": \\\"67d8fd4b-85de-4d57-a829-7ce0bf268c5d\\\",\\n  \\\"status\\\": \\\"failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentStackDeploymentFailed\\\",\\n    \\\"message\\\": \\\"One or more resources could not be deployed. Correlation id: '676c3fe4-b0a1-44f5-8602-c3c05012b379'.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9428224-mgmt-2/providers/Microsoft.Resources/deployments/l4mv3scto7z6edigthkjdr62jbdrgoxoif33nvakxlwvg-260824084ipd0\\\",\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n        \\\"details\\\": [\\n          {\\n            \\\"code\\\": \\\"OverconstrainedZonalAllocationRequest\\\",\\n            \\\"message\\\": \\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9428224-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-41975975-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 3, 2), 'Standard_E8-4s_v6' (zones 3, 2), 'Standard_E8-4ds_v6' (zones 2, 3) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9428224-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-infra1-41975975-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n  - Availability Zone\\\\n  - Differencing (Ephemeral) Disks\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n  - VM Size\\\\n\\\"\\n          }\\n        ]\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6447/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091793135989428224","occurred_at":"2026-08-24T07:42:01Z","signature_id":"497ab79e6fe621da7f982b21f5dd28a45dc2f63297d3ed53abd84200879145f5","pr_number":6447,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"92864eb284798289fed5d474f4e88228a7f26e3d9255d12a9e66635826bdc5ce","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '1' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-18T01:47:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:126]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004e15e0\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-wchrx8wtt6zv\" to finish creating: GET https://rp.j5184000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a62a8a64-d466-4b4b-aecf-3fa47402db53\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a62a8a64-d466-4b4b-aecf-3fa47402db53\",\n      \"name\": \"a62a8a64-d466-4b4b-aecf-3fa47402db53\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T02:49:55.713363479Z\",\n      \"endTime\": \"2026-08-18T02:50:32.796114827Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-wchrx8wtt6zv/providers/Microsoft.Compute/diskEncryptionSets/des-encrypt-des\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:126]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004e15e0\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-wchrx8wtt6zv\" to finish creating: GET https://rp.j5184000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a62a8a64-d466-4b4b-aecf-3fa47402db53\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a62a8a64-d466-4b4b-aecf-3fa47402db53\",\n      \"name\": \"a62a8a64-d466-4b4b-aecf-3fa47402db53\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T02:49:55.713363479Z\",\n      \"endTime\": \"2026-08-18T02:50:32.796114827Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-wchrx8wtt6zv/providers/Microsoft.Compute/diskEncryptionSets/des-encrypt-des\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6351/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089529602845184000","occurred_at":"2026-08-18T01:47:33Z","signature_id":"8c948f490630ad31b777070c3af68d2453cf07835bcbe82994409a12a915940a","pr_number":6351,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0fa837367648e2c6015295472e0f8df164e966eed931bdff87d5784d41e8c7f8","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-18T14:39:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-22-w7j5lmd99lw9/cluster-candidate-4-22-wqjj4v should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ec6420\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-wqjj4v: failed waiting for cluster=\"cluster-candidate-4-22-wqjj4v\" in resourcegroup=\"rg-candidate-4-22-w7j5lmd99lw9\" to finish creating: GET https://rp.j4002688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91e78729-9c8b-4837-b05a-68835b5664a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91e78729-9c8b-4837-b05a-68835b5664a2\",\n      \"name\": \"91e78729-9c8b-4837-b05a-68835b5664a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T15:31:32.438526546Z\",\n      \"endTime\": \"2026-08-18T15:50:34.901509202Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-22-w7j5lmd99lw9/cluster-candidate-4-22-wqjj4v should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ec6420\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-wqjj4v: failed waiting for cluster=\"cluster-candidate-4-22-wqjj4v\" in resourcegroup=\"rg-candidate-4-22-w7j5lmd99lw9\" to finish creating: GET https://rp.j4002688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91e78729-9c8b-4837-b05a-68835b5664a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91e78729-9c8b-4837-b05a-68835b5664a2\",\n      \"name\": \"91e78729-9c8b-4837-b05a-68835b5664a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T15:31:32.438526546Z\",\n      \"endTime\": \"2026-08-18T15:50:34.901509202Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089723779814002688","occurred_at":"2026-08-18T14:39:04Z","signature_id":"ad545a5888ee01721c3841fb9536deb80d53fb55f352cc8d9991374cccb98041","pr_number":6157,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"da0050d2b241d5b64bca4e3ea0c184bfa4a846f096a3f477d0e09b223d10f5d6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-22T00:10:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-wpfzf9vnbsdw/cluster-candidate-4-20-mrfd9x should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010867c0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-mrfd9x: failed waiting for cluster=\"cluster-candidate-4-20-mrfd9x\" in resourcegroup=\"rg-candidate-4-20-wpfzf9vnbsdw\" to finish creating: GET https://rp.j1593344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c796e99-5d3d-4dfc-bde8-3e8f96015f93\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c796e99-5d3d-4dfc-bde8-3e8f96015f93\",\n      \"name\": \"7c796e99-5d3d-4dfc-bde8-3e8f96015f93\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:17:53.391052013Z\",\n      \"endTime\": \"2026-08-22T01:37:02.44294507Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-wpfzf9vnbsdw/cluster-candidate-4-20-mrfd9x should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010867c0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-mrfd9x: failed waiting for cluster=\"cluster-candidate-4-20-mrfd9x\" in resourcegroup=\"rg-candidate-4-20-wpfzf9vnbsdw\" to finish creating: GET https://rp.j1593344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c796e99-5d3d-4dfc-bde8-3e8f96015f93\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7c796e99-5d3d-4dfc-bde8-3e8f96015f93\",\n      \"name\": \"7c796e99-5d3d-4dfc-bde8-3e8f96015f93\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:17:53.391052013Z\",\n      \"endTime\": \"2026-08-22T01:37:02.44294507Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090954745601593344","occurred_at":"2026-08-22T00:10:34Z","signature_id":"0f615daad0c6c3c41aba5ffc09456a12dc1b17c512cf6fcba2046e7323cdd028","pr_number":6593,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"222389f191e96faa19cc745a6598bb23f8837d217b21c6acf614c953420d05f6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[2,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-19T18:39:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--h25qw2r4rdvw/cluster-hs-pre-kkhnd8 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010a0700\u003e: \n    failed to create HCP cluster cluster-hs-pre-kkhnd8: failed waiting for cluster=\"cluster-hs-pre-kkhnd8\" in resourcegroup=\"rg-hs-pre--h25qw2r4rdvw\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e1bf450-7343-497d-89ed-b637b8ac274d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e1bf450-7343-497d-89ed-b637b8ac274d\",\n      \"name\": \"7e1bf450-7343-497d-89ed-b637b8ac274d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:28:09.436312335Z\",\n      \"endTime\": \"2026-08-19T19:47:09.82470814Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--h25qw2r4rdvw/cluster-hs-pre-kkhnd8 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010a0700\u003e: \n    failed to create HCP cluster cluster-hs-pre-kkhnd8: failed waiting for cluster=\"cluster-hs-pre-kkhnd8\" in resourcegroup=\"rg-hs-pre--h25qw2r4rdvw\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e1bf450-7343-497d-89ed-b637b8ac274d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7e1bf450-7343-497d-89ed-b637b8ac274d\",\n      \"name\": \"7e1bf450-7343-497d-89ed-b637b8ac274d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:28:09.436312335Z\",\n      \"endTime\": \"2026-08-19T19:47:09.82470814Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"c7d630f0a28c9b8d1aeb7eeaccfb13c753fc80a06498686e5dd2af7ee7790d98","pr_number":6599,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"8a80f07fe2ebdec087cc11f27aeb724cf8c6cf8aa029e8f467e1f89fd185112d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-network-operator, ingress-operator; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-22T00:25:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-kkp4qflqb9tl/cluster-candidate-5-0-nzjftt should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f14860\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-nzjftt: failed waiting for cluster=\"cluster-candidate-5-0-nzjftt\" in resourcegroup=\"rg-candidate-5-0-kkp4qflqb9tl\" to finish creating: GET https://rp.j3592192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/879d27b2-7910-474a-ade0-ae3e7eab5f6e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/879d27b2-7910-474a-ade0-ae3e7eab5f6e\",\n      \"name\": \"879d27b2-7910-474a-ade0-ae3e7eab5f6e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:19:04.482166059Z\",\n      \"endTime\": \"2026-08-22T01:38:11.22344033Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, cluster-network-operator\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-kkp4qflqb9tl/cluster-candidate-5-0-nzjftt should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f14860\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-nzjftt: failed waiting for cluster=\"cluster-candidate-5-0-nzjftt\" in resourcegroup=\"rg-candidate-5-0-kkp4qflqb9tl\" to finish creating: GET https://rp.j3592192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/879d27b2-7910-474a-ade0-ae3e7eab5f6e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/879d27b2-7910-474a-ade0-ae3e7eab5f6e\",\n      \"name\": \"879d27b2-7910-474a-ade0-ae3e7eab5f6e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T01:19:04.482166059Z\",\n      \"endTime\": \"2026-08-22T01:38:11.22344033Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, cluster-network-operator\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6554/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090958470403592192","occurred_at":"2026-08-22T00:25:22Z","signature_id":"201d37275ac85dc0beaa94be1f84bc640b3be944f1830b8c0efc34b279522a24","pr_number":6554,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"de9ee73e1f42212ffe8bacf6081359c2c391b2c47c68ba07fa4cd11f00e7b015","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T17:48:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001314280\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-f7brrnlwp2l5\" to finish creating: GET https://rp.j8125696.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1e6d169-deac-4e56-b97f-0a253becce60\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1e6d169-deac-4e56-b97f-0a253becce60\",\n      \"name\": \"e1e6d169-deac-4e56-b97f-0a253becce60\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T18:37:36.102317205Z\",\n      \"endTime\": \"2026-08-20T18:56:41.667600628Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, packageserver, redhat-operators-catalog, capi-provider, cluster-version-operator, cluster-storage-operator, dns-operator, olm-operator, cluster-policy-controller, hosted-cluster-config-operator, cluster-network-operator, ingress-operator, kube-scheduler, csi-snapshot-controller-operator, kube-controller-manager, openshift-route-controller-manager, cluster-image-registry-operator, olm-collect-profiles, community-operators-catalog, cluster-node-tuning-operator, redhat-marketplace-catalog, certified-operators-catalog, machine-approver, openshift-apiserver, azure-cloud-controller-manager, ignition-server-proxy, cluster-autoscaler, control-plane-pki-operator, openshift-controller-manager, konnectivity-agent, ignition-server; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001314280\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-f7brrnlwp2l5\" to finish creating: GET https://rp.j8125696.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1e6d169-deac-4e56-b97f-0a253becce60\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1e6d169-deac-4e56-b97f-0a253becce60\",\n      \"name\": \"e1e6d169-deac-4e56-b97f-0a253becce60\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T18:37:36.102317205Z\",\n      \"endTime\": \"2026-08-20T18:56:41.667600628Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, packageserver, redhat-operators-catalog, capi-provider, cluster-version-operator, cluster-storage-operator, dns-operator, olm-operator, cluster-policy-controller, hosted-cluster-config-operator, cluster-network-operator, ingress-operator, kube-scheduler, csi-snapshot-controller-operator, kube-controller-manager, openshift-route-controller-manager, cluster-image-registry-operator, olm-collect-profiles, community-operators-catalog, cluster-node-tuning-operator, redhat-marketplace-catalog, certified-operators-catalog, machine-approver, openshift-apiserver, azure-cloud-controller-manager, ignition-server-proxy, cluster-autoscaler, control-plane-pki-operator, openshift-controller-manager, konnectivity-agent, ignition-server; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"d31df3c1d2280d7555d7fa65f94d10af826e62abc345c39568cdb055792e220d","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d2829443a4ebdae768869c26f5c53cd707f73b967a4f58e299f0ab62d07b2457","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, cluster-image-registry-operator, cluster-network-operator, community-operators-catalog, ingress-operator, olm-operator, packageserver, redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-22T10:03:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000da51e0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-d746nz7xlk2s\" to finish creating: GET https://rp.j3428992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5fe3108e-0452-4336-9476-4c9bc112740c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5fe3108e-0452-4336-9476-4c9bc112740c\",\n      \"name\": \"5fe3108e-0452-4336-9476-4c9bc112740c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T10:45:48.497379279Z\",\n      \"endTime\": \"2026-08-22T11:04:48.540876115Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, packageserver, openshift-controller-manager, cluster-policy-controller, dns-operator, certified-operators-catalog, cluster-node-tuning-operator, csi-snapshot-controller-operator, openshift-route-controller-manager, ingress-operator, cluster-image-registry-operator, catalog-operator, olm-operator, cluster-version-operator, hosted-cluster-config-operator, community-operators-catalog, cluster-network-operator, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000da51e0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-d746nz7xlk2s\" to finish creating: GET https://rp.j3428992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5fe3108e-0452-4336-9476-4c9bc112740c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5fe3108e-0452-4336-9476-4c9bc112740c\",\n      \"name\": \"5fe3108e-0452-4336-9476-4c9bc112740c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T10:45:48.497379279Z\",\n      \"endTime\": \"2026-08-22T11:04:48.540876115Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-storage-operator, packageserver, openshift-controller-manager, cluster-policy-controller, dns-operator, certified-operators-catalog, cluster-node-tuning-operator, csi-snapshot-controller-operator, openshift-route-controller-manager, ingress-operator, cluster-image-registry-operator, catalog-operator, olm-operator, cluster-version-operator, hosted-cluster-config-operator, community-operators-catalog, cluster-network-operator, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091103857353428992","occurred_at":"2026-08-22T10:03:05Z","signature_id":"dc24633d9e5d820847c90cb66faabc819f731ccb727551969b9a5ee3eb7f3c9b","pr_number":6651,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"42b2d5dca43e9e034a17b7b121f324959a41ced09de1009bebe039aef5a32bf2","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog, cluster-image-registry-operator, cluster-node-tuning-operator, cluster-policy-controller, csi-snapshot-controller-operator, ignition-server-proxy, ingress-operator, redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T18:39:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001004c60\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-2mn5zf9jcbq8\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80ff098e-3bea-479a-9923-3c6d996be86d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80ff098e-3bea-479a-9923-3c6d996be86d\",\n      \"name\": \"80ff098e-3bea-479a-9923-3c6d996be86d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:28:41.976383533Z\",\n      \"endTime\": \"2026-08-19T19:47:49.788162245Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, cluster-node-tuning-operator, cluster-image-registry-operator, redhat-operators-catalog, ingress-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001004c60\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-2mn5zf9jcbq8\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80ff098e-3bea-479a-9923-3c6d996be86d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/80ff098e-3bea-479a-9923-3c6d996be86d\",\n      \"name\": \"80ff098e-3bea-479a-9923-3c6d996be86d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:28:41.976383533Z\",\n      \"endTime\": \"2026-08-19T19:47:49.788162245Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, cluster-node-tuning-operator, cluster-image-registry-operator, redhat-operators-catalog, ingress-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"97c529fbcfca706ccd9d033fcefb76e6725645f13e56e9dbe948031baa2eefc4","pr_number":6599,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"8a2bad4325413455e9a17bdb9d76f2dce3a57579df59da3db41f3afd4283d135","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, csi-snapshot-controller-operator, ingress-operator, redhat-marketplace-catalog, redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T18:39:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00134aa60\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-5ctgfw4z77n6\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/375f0981-14a8-4380-a7e8-1e4162f764ae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/375f0981-14a8-4380-a7e8-1e4162f764ae\",\n      \"name\": \"375f0981-14a8-4380-a7e8-1e4162f764ae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:29:02.748319627Z\",\n      \"endTime\": \"2026-08-19T19:48:09.81648964Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, redhat-operators-catalog, csi-snapshot-controller-operator, ingress-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00134aa60\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-5ctgfw4z77n6\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/375f0981-14a8-4380-a7e8-1e4162f764ae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/375f0981-14a8-4380-a7e8-1e4162f764ae\",\n      \"name\": \"375f0981-14a8-4380-a7e8-1e4162f764ae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:29:02.748319627Z\",\n      \"endTime\": \"2026-08-19T19:48:09.81648964Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, redhat-operators-catalog, csi-snapshot-controller-operator, ingress-operator, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"15c8cd6151771fba631053e4c516e39b3df9fdc4db1e62da2330bcefb8214b01","pr_number":6599,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"4bd062f4cae165ed4f3c87f0a702d7aa278063a030bc7da353586a552446d86b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-18T14:39:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a9f3a0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-9bmrqlgglf5f\" to finish creating: GET https://rp.j4002688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19df74a2-3021-4f92-8f1b-ff3798b18a77\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19df74a2-3021-4f92-8f1b-ff3798b18a77\",\n      \"name\": \"19df74a2-3021-4f92-8f1b-ff3798b18a77\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T15:32:29.097143931Z\",\n      \"endTime\": \"2026-08-18T15:51:35.137229311Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a9f3a0\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-9bmrqlgglf5f\" to finish creating: GET https://rp.j4002688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19df74a2-3021-4f92-8f1b-ff3798b18a77\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19df74a2-3021-4f92-8f1b-ff3798b18a77\",\n      \"name\": \"19df74a2-3021-4f92-8f1b-ff3798b18a77\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T15:32:29.097143931Z\",\n      \"endTime\": \"2026-08-18T15:51:35.137229311Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089723779814002688","occurred_at":"2026-08-18T14:39:04Z","signature_id":"9915d614d3b35eec4dbbb8f550a33792299b44edf15895e6075d20a91526f4f9","pr_number":6157,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f7ad14e1158d7d53f17395adc1367d8f6b3116098187de178cccafef487d5b29","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-22T10:03:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a5f0a0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-pp8z58bxmcb4\" to finish creating: GET https://rp.j3428992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2499d56f-998f-453a-9516-ce8ca4b2d871\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2499d56f-998f-453a-9516-ce8ca4b2d871\",\n      \"name\": \"2499d56f-998f-453a-9516-ce8ca4b2d871\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T10:46:33.945954582Z\",\n      \"endTime\": \"2026-08-22T11:05:38.525996595Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a5f0a0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-pp8z58bxmcb4\" to finish creating: GET https://rp.j3428992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2499d56f-998f-453a-9516-ce8ca4b2d871\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2499d56f-998f-453a-9516-ce8ca4b2d871\",\n      \"name\": \"2499d56f-998f-453a-9516-ce8ca4b2d871\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T10:46:33.945954582Z\",\n      \"endTime\": \"2026-08-22T11:05:38.525996595Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091103857353428992","occurred_at":"2026-08-22T10:03:05Z","signature_id":"c21db095d9a7b7ff0f65acbc17c905f119cc2e8936e61c276d6fb4034c19e49c","pr_number":6651,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b652815635f16d983d1325dbd7e304560647aed7cf1be6254e8a2a57faf1d4fa","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ef90e0\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-2zg88lxq9q4w\" to finish creating: GET https://rp.j3558656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b976f9f3-2891-4433-868a-76a8d23de317\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b976f9f3-2891-4433-868a-76a8d23de317\",\n      \"name\": \"b976f9f3-2891-4433-868a-76a8d23de317\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T16:54:08.369328068Z\",\n      \"endTime\": \"2026-08-24T17:13:08.208412685Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ef90e0\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-2zg88lxq9q4w\" to finish creating: GET https://rp.j3558656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b976f9f3-2891-4433-868a-76a8d23de317\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b976f9f3-2891-4433-868a-76a8d23de317\",\n      \"name\": \"b976f9f3-2891-4433-868a-76a8d23de317\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T16:54:08.369328068Z\",\n      \"endTime\": \"2026-08-24T17:13:08.208412685Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091920407983558656","occurred_at":"2026-08-24T16:06:36Z","signature_id":"fdde03128e739a2f562ab5c3485dae63a0f561b3b4659197733a62ff1ea2a4e8","pr_number":6584,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"3007a8035cb9883551a2dd738a6538996a11b3ee85467615faaa28d2c01efb23","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \"\u003curl\u003e\": EOF; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T17:48:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012ac800\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-k2z986tbg59c\" to finish creating: GET https://rp.j8125696.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f045696f-4b84-45c9-977c-312bcc37c82d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f045696f-4b84-45c9-977c-312bcc37c82d\",\n      \"name\": \"f045696f-4b84-45c9-977c-312bcc37c82d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T18:37:34.536304881Z\",\n      \"endTime\": \"2026-08-20T18:56:41.61282925Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.x7u1r2j3j5y3g7s.amf4.j8125696.hcp.osadev.cloud:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 2 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012ac800\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-k2z986tbg59c\" to finish creating: GET https://rp.j8125696.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f045696f-4b84-45c9-977c-312bcc37c82d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f045696f-4b84-45c9-977c-312bcc37c82d\",\n      \"name\": \"f045696f-4b84-45c9-977c-312bcc37c82d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T18:37:34.536304881Z\",\n      \"endTime\": \"2026-08-20T18:56:41.61282925Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.x7u1r2j3j5y3g7s.amf4.j8125696.hcp.osadev.cloud:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 2 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 2 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"1300ae1779ab2d89390be4908e232d8c8bb56305577ccb0f64ff01c7791b7baf","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"114cb61901859f036a8d83d4edb4405b6736c8fe33bb88a26c3df9e5b3d78697","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingOnInfrastructureReady: Cluster infrastructure is still provisioning; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-19T22:35:50Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a84820\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-g742bqj9glvt\" to finish creating: GET https://rp.j4815104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b362542a-ad60-4297-b5da-8467abaa215b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b362542a-ad60-4297-b5da-8467abaa215b\",\n      \"name\": \"b362542a-ad60-4297-b5da-8467abaa215b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T23:20:10.360400826Z\",\n      \"endTime\": \"2026-08-19T23:39:12.433437262Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingOnInfrastructureReady: Cluster infrastructure is still provisioning; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a84820\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-g742bqj9glvt\" to finish creating: GET https://rp.j4815104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b362542a-ad60-4297-b5da-8467abaa215b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b362542a-ad60-4297-b5da-8467abaa215b\",\n      \"name\": \"b362542a-ad60-4297-b5da-8467abaa215b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T23:20:10.360400826Z\",\n      \"endTime\": \"2026-08-19T23:39:12.433437262Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: WaitingOnInfrastructureReady: Cluster infrastructure is still provisioning; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090206132864815104","occurred_at":"2026-08-19T22:35:50Z","signature_id":"4885cbb4cc6da23b560a2111d8dfc0586a851d3c87fe24afcb3a3acbe0789700","pr_number":6607,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"07c5a276cc557395833fa5a9a3dd00ed9d007140f2aa66df2672b994256306c7","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[1,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-19T18:39:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001d84120\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-dkq6w72s7qx7\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56c87b3e-1932-4c66-9b4c-72604659dde3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56c87b3e-1932-4c66-9b4c-72604659dde3\",\n      \"name\": \"56c87b3e-1932-4c66-9b4c-72604659dde3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:28:22.919700021Z\",\n      \"endTime\": \"2026-08-19T19:47:29.851293041Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001d84120\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-dkq6w72s7qx7\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56c87b3e-1932-4c66-9b4c-72604659dde3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/56c87b3e-1932-4c66-9b4c-72604659dde3\",\n      \"name\": \"56c87b3e-1932-4c66-9b4c-72604659dde3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:28:22.919700021Z\",\n      \"endTime\": \"2026-08-19T19:47:29.851293041Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"dccfd57f9593df2be76271b9211656e5ac05b45c64d18576620ed8bc61afd481","pr_number":6599,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"6c556b8d011e6b7ebbaeb30df8771947fcda1d72a66e01bdb4df4cabd80a8a4e","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[1,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-18T18:48:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f982e0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fc4w4fpvm56p\" to finish creating: GET https://rp.j8620160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ba2a479e-dd5d-4ea4-95c2-5e753791256d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ba2a479e-dd5d-4ea4-95c2-5e753791256d\",\n      \"name\": \"ba2a479e-dd5d-4ea4-95c2-5e753791256d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T19:40:03.054838398Z\",\n      \"endTime\": \"2026-08-18T19:59:04.701867285Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f982e0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fc4w4fpvm56p\" to finish creating: GET https://rp.j8620160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ba2a479e-dd5d-4ea4-95c2-5e753791256d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ba2a479e-dd5d-4ea4-95c2-5e753791256d\",\n      \"name\": \"ba2a479e-dd5d-4ea4-95c2-5e753791256d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T19:40:03.054838398Z\",\n      \"endTime\": \"2026-08-18T19:59:04.701867285Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 2 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089786820278620160","occurred_at":"2026-08-18T18:48:55Z","signature_id":"e66b64e899dd50b8229507b5761dedf198ef061c357fb29516e4036a8c141d5a","pr_number":6492,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e5527c3b285a31542ef0763819720c5a2c3854f85d54e55990c34e930b91ed3a","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, kube-controller-manager, openshift-apiserver, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-18T11:44:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-5-0-tcq7ssv5jp2b/cluster-candidate-5-0-5llz78 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f067a0\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-5llz78: failed waiting for cluster=\"cluster-candidate-5-0-5llz78\" in resourcegroup=\"rg-candidate-5-0-tcq7ssv5jp2b\" to finish creating: GET https://rp.j9813120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b53bd407-1e3e-453d-ac75-e30d7648200f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b53bd407-1e3e-453d-ac75-e30d7648200f\",\n      \"name\": \"b53bd407-1e3e-453d-ac75-e30d7648200f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T12:38:35.542182012Z\",\n      \"endTime\": \"2026-08-18T12:57:38.27022929Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-5-0-tcq7ssv5jp2b/cluster-candidate-5-0-5llz78 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f067a0\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-5llz78: failed waiting for cluster=\"cluster-candidate-5-0-5llz78\" in resourcegroup=\"rg-candidate-5-0-tcq7ssv5jp2b\" to finish creating: GET https://rp.j9813120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b53bd407-1e3e-453d-ac75-e30d7648200f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b53bd407-1e3e-453d-ac75-e30d7648200f\",\n      \"name\": \"b53bd407-1e3e-453d-ac75-e30d7648200f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T12:38:35.542182012Z\",\n      \"endTime\": \"2026-08-18T12:57:38.27022929Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089679732839813120","occurred_at":"2026-08-18T11:44:07Z","signature_id":"72a23bc947e208019444008b323b8a7d3651c93d4984d2daef20416d2aef4385","pr_number":6553,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"406e5cbdda56870662bb8dffe74cefec7ec1bd02d0ab40f75b9206c9de3f1748","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-18T11:44:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001288620\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-8hmxgjm8zgzt\" to finish creating: GET https://rp.j9813120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38a18e49-3d0a-4431-bf5c-9586aeba70a0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38a18e49-3d0a-4431-bf5c-9586aeba70a0\",\n      \"name\": \"38a18e49-3d0a-4431-bf5c-9586aeba70a0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T12:38:58.93809557Z\",\n      \"endTime\": \"2026-08-18T12:57:58.01101578Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001288620\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-8hmxgjm8zgzt\" to finish creating: GET https://rp.j9813120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38a18e49-3d0a-4431-bf5c-9586aeba70a0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38a18e49-3d0a-4431-bf5c-9586aeba70a0\",\n      \"name\": \"38a18e49-3d0a-4431-bf5c-9586aeba70a0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T12:38:58.93809557Z\",\n      \"endTime\": \"2026-08-18T12:57:58.01101578Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089679732839813120","occurred_at":"2026-08-18T11:44:07Z","signature_id":"11340b78a69c03acc992040afbaab2d3854033f56be0e16a99d0202103c9f52b","pr_number":6553,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"46dc8f1bee0f9d9e31cc6b9183855a5e855411a91fb0abc98ec48dbf9a7eea53","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-18T11:44:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-22-pt6fgpnb2b75/cluster-candidate-4-22-txhq7g should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dce120\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-txhq7g: failed waiting for cluster=\"cluster-candidate-4-22-txhq7g\" in resourcegroup=\"rg-candidate-4-22-pt6fgpnb2b75\" to finish creating: GET https://rp.j9813120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/442dc1eb-0ba2-46fb-8e59-b14a67c2090b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/442dc1eb-0ba2-46fb-8e59-b14a67c2090b\",\n      \"name\": \"442dc1eb-0ba2-46fb-8e59-b14a67c2090b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T12:38:36.789457078Z\",\n      \"endTime\": \"2026-08-18T12:57:38.053407648Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-22-pt6fgpnb2b75/cluster-candidate-4-22-txhq7g should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dce120\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-txhq7g: failed waiting for cluster=\"cluster-candidate-4-22-txhq7g\" in resourcegroup=\"rg-candidate-4-22-pt6fgpnb2b75\" to finish creating: GET https://rp.j9813120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/442dc1eb-0ba2-46fb-8e59-b14a67c2090b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/442dc1eb-0ba2-46fb-8e59-b14a67c2090b\",\n      \"name\": \"442dc1eb-0ba2-46fb-8e59-b14a67c2090b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T12:38:36.789457078Z\",\n      \"endTime\": \"2026-08-18T12:57:38.053407648Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089679732839813120","occurred_at":"2026-08-18T11:44:07Z","signature_id":"d80a703b2984229683de5b48ac8412427d74f3065b98e3ef651c160ad66e1bf2","pr_number":6553,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"63b932f574b85083f97c91322b9a6ecdafe1d1b810c184f1491c16896b37be79","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T18:39:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7cce0\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-zqvlg88vd2jj\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0c9c39b-413e-43f6-bf72-32d04b43d23e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0c9c39b-413e-43f6-bf72-32d04b43d23e\",\n      \"name\": \"b0c9c39b-413e-43f6-bf72-32d04b43d23e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:28:40.237300396Z\",\n      \"endTime\": \"2026-08-19T19:47:49.770337967Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: hosted-cluster-config-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7cce0\u003e: \n    failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-zqvlg88vd2jj\" to finish creating: GET https://rp.j1099648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0c9c39b-413e-43f6-bf72-32d04b43d23e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0c9c39b-413e-43f6-bf72-32d04b43d23e\",\n      \"name\": \"b0c9c39b-413e-43f6-bf72-32d04b43d23e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T19:28:40.237300396Z\",\n      \"endTime\": \"2026-08-19T19:47:49.770337967Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: hosted-cluster-config-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090146594451099648","occurred_at":"2026-08-19T18:39:15Z","signature_id":"6cb42325713d6fee4731d031f5a77e75d73f77401d53264f8af766970b2dfe91","pr_number":6599,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"676e68f8a68616ac8c57dc0fe075f4a0d00c211dff572a92461d02984b57a771","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-18T11:44:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015e6260\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-qdb8jmfps2h5\" to finish creating: GET https://rp.j9813120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/494af206-9f47-4fa6-aa86-9ff4bb0dad1b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/494af206-9f47-4fa6-aa86-9ff4bb0dad1b\",\n      \"name\": \"494af206-9f47-4fa6-aa86-9ff4bb0dad1b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T12:38:35.540517856Z\",\n      \"endTime\": \"2026-08-18T12:57:38.071517574Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, openshift-route-controller-manager, kube-scheduler, cluster-policy-controller, ignition-server, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015e6260\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-qdb8jmfps2h5\" to finish creating: GET https://rp.j9813120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/494af206-9f47-4fa6-aa86-9ff4bb0dad1b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/494af206-9f47-4fa6-aa86-9ff4bb0dad1b\",\n      \"name\": \"494af206-9f47-4fa6-aa86-9ff4bb0dad1b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T12:38:35.540517856Z\",\n      \"endTime\": \"2026-08-18T12:57:38.071517574Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, openshift-route-controller-manager, kube-scheduler, cluster-policy-controller, ignition-server, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089679732839813120","occurred_at":"2026-08-18T11:44:07Z","signature_id":"c13ad22477acfc11cc9c88996da8292c9ecfb955ce0b557b665484e4d48c0b84","pr_number":6553,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d750117ee8ebeaf1ff99e5c34b4cf84643ef0acdb6a9d1ab17022db0a4dd4379","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: externalAuths, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[1,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-19T14:43:14Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00149e1f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-4jhgzkdbjlgs\" to finish deleting: GET https://rp.j2752384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38155a97-ba59-4f8d-8be4-28e12f30ac13\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38155a97-ba59-4f8d-8be4-28e12f30ac13\",\n      \"name\": \"38155a97-ba59-4f8d-8be4-28e12f30ac13\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T16:04:26.177378827Z\",\n      \"endTime\": \"2026-08-19T16:28:27.080589493Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9ns3hqlu46g55l00aeovpib48f618n still exists (deletion dispatched at 2026-08-19T16:04:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00149e1f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-4jhgzkdbjlgs\" to finish deleting: GET https://rp.j2752384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38155a97-ba59-4f8d-8be4-28e12f30ac13\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38155a97-ba59-4f8d-8be4-28e12f30ac13\",\n      \"name\": \"38155a97-ba59-4f8d-8be4-28e12f30ac13\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T16:04:26.177378827Z\",\n      \"endTime\": \"2026-08-19T16:28:27.080589493Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9ns3hqlu46g55l00aeovpib48f618n still exists (deletion dispatched at 2026-08-19T16:04:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090087196852752384","occurred_at":"2026-08-19T14:43:14Z","signature_id":"0070970079376a5ff3304016bcd4e59257420a882773ccd09b7df9dc62e41353","pr_number":6608,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"da9b2481b911b0663d057a478fe773143491e6eb267de20ae556db13d4d07be0","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: applydesires, externalAuths, nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-18T12:07:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00181c198\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-r5xqsnslvnr7\" to finish deleting: GET https://rp.j0479744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4049de26-4e01-4e10-8f66-700763cbd91f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4049de26-4e01-4e10-8f66-700763cbd91f\",\n      \"name\": \"4049de26-4e01-4e10-8f66-700763cbd91f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T13:22:58.557109512Z\",\n      \"endTime\": \"2026-08-18T13:47:04.744384116Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s90fh4q7oruri74ihm8jelqcslg1fot still exists (deletion dispatched at 2026-08-18T13:22:58Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 13 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00181c198\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-r5xqsnslvnr7\" to finish deleting: GET https://rp.j0479744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4049de26-4e01-4e10-8f66-700763cbd91f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4049de26-4e01-4e10-8f66-700763cbd91f\",\n      \"name\": \"4049de26-4e01-4e10-8f66-700763cbd91f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T13:22:58.557109512Z\",\n      \"endTime\": \"2026-08-18T13:47:04.744384116Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s90fh4q7oruri74ihm8jelqcslg1fot still exists (deletion dispatched at 2026-08-18T13:22:58Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 13 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089685516600479744","occurred_at":"2026-08-18T12:07:05Z","signature_id":"c83f917bc3bfba30f3f1b7b8e344ab541e830439375a54a3ba983a78f9559f35","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ba89a4166305f6c19c8eaadc8056c4375814afedeefcd4ac9c286060c7d5cad1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-10"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-14T20:23:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000ebc2a0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-h4ldj6\" in resourcegroup=\"rg-candidate-5-0-xjnldggtrt7m\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa0102ea-068c-4907-92f2-69579c623cac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa0102ea-068c-4907-92f2-69579c623cac\",\n      \"name\": \"fa0102ea-068c-4907-92f2-69579c623cac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:21:51.698922375Z\",\n      \"endTime\": \"2026-08-24T15:45:59.987785659Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tm5ibpilvjmdnr77hbqf9t255bon still exists (deletion dispatched at 2026-08-24T15:21:51Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000ebc2a0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-h4ldj6\" in resourcegroup=\"rg-candidate-5-0-xjnldggtrt7m\" to finish deleting: GET https://rp.j9012096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa0102ea-068c-4907-92f2-69579c623cac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa0102ea-068c-4907-92f2-69579c623cac\",\n      \"name\": \"fa0102ea-068c-4907-92f2-69579c623cac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T15:21:51.698922375Z\",\n      \"endTime\": \"2026-08-24T15:45:59.987785659Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd0tm5ibpilvjmdnr77hbqf9t255bon still exists (deletion dispatched at 2026-08-24T15:21:51Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091892836139012096","occurred_at":"2026-08-24T14:18:07Z","signature_id":"60c407c2c17844ded754b0d5e8dda40f125391008bbbdd669c23fa4348b5b316","pr_number":6232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4bb3b677e4e47964490ee71786cbc7256b98260157b7c34447e1a99e7df57e85","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[1,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-18T19:29:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001162480\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-hzhh65\" in resourcegroup=\"rg-np-skip-minor-hzhh65-cxh8rqw8dwrz\" to finish deleting: GET https://rp.j3222016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de1cfe84-4963-47bc-9590-189164cc3a40\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de1cfe84-4963-47bc-9590-189164cc3a40\",\n      \"name\": \"de1cfe84-4963-47bc-9590-189164cc3a40\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T20:56:15.822318952Z\",\n      \"endTime\": \"2026-08-18T21:20:19.68093928Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001162480\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-hzhh65\" in resourcegroup=\"rg-np-skip-minor-hzhh65-cxh8rqw8dwrz\" to finish deleting: GET https://rp.j3222016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de1cfe84-4963-47bc-9590-189164cc3a40\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de1cfe84-4963-47bc-9590-189164cc3a40\",\n      \"name\": \"de1cfe84-4963-47bc-9590-189164cc3a40\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T20:56:15.822318952Z\",\n      \"endTime\": \"2026-08-18T21:20:19.68093928Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089796885723222016","occurred_at":"2026-08-18T19:29:38Z","signature_id":"ac409d0c93e08bed6e87db1bbd614f69372f7ce28513667a12dc216382d85334","pr_number":6592,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"dfea829247c76b95c4c16a453220bb1a88418327719d31f8f97dda000a27dc8f","failure_pattern":"ERROR CODE: NotFound; detail message HCP resource requirements \"default\" not found","search_query":"\"message\": \"HCP resource requirements \\\"default\\\" not found\"","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Fleet should have HCP resource requirements data","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-18T11:44:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Fleet should have HCP resource requirements data","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:115]: Timed out after 1800.001s.\nHCP resource requirements were not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:102 with:\nfailed to get HCP resource requirements\nUnexpected error:\n    \u003c*errors.errorString | 0xc000bb6960\u003e: \n    expected status 200 OK, got 404: {\n        \"error\": {\n            \"code\": \"NotFound\",\n            \"message\": \"HCP resource requirements \\\"default\\\" not found\"\n        }\n    }\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:115]: Timed out after 1800.001s.\nHCP resource requirements were not populated in time\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/fleet_registration.go:102 with:\nfailed to get HCP resource requirements\nUnexpected error:\n    \u003c*errors.errorString | 0xc000bb6960\u003e: \n    expected status 200 OK, got 404: {\n        \"error\": {\n            \"code\": \"NotFound\",\n            \"message\": \"HCP resource requirements \\\"default\\\" not found\"\n        }\n    }\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6553/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089679732839813120","occurred_at":"2026-08-18T11:44:07Z","signature_id":"2078f17784bb4e658cb1ac8b66951a5523a9b11b2c53f97caf496e083ffea702","pr_number":6553,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"80fea695ea79e0aa0c93681c7b691eabdeba791c0a06933a05c165daa9a55413","failure_pattern":"Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"Get \"https://api.a3v1j8m2n0v6k1m.w13o.j2626944.hcp.osadev.cloud:443/apis/config.openshift.io/v1/clusterversions/version\": dial tcp: lookup api.a3v1j8m2n0v6k1m.w13o.j2626944.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.130.232.21:49833-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-18T22:34:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:166]: failed to get ClusterVersion resource\nUnexpected error:\n    \u003c*url.Error | 0xc000dc0d50\u003e: \n    Get \"https://api.a3v1j8m2n0v6k1m.w13o.j2626944.hcp.osadev.cloud:443/apis/config.openshift.io/v1/clusterversions/version\": dial tcp: lookup api.a3v1j8m2n0v6k1m.w13o.j2626944.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.130.232.21:49833-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:166]: failed to get ClusterVersion resource\nUnexpected error:\n    \u003c*url.Error | 0xc000dc0d50\u003e: \n    Get \"https://api.a3v1j8m2n0v6k1m.w13o.j2626944.hcp.osadev.cloud:443/apis/config.openshift.io/v1/clusterversions/version\": dial tcp: lookup api.a3v1j8m2n0v6k1m.w13o.j2626944.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.130.232.21:49833-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6557/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089843361052626944","occurred_at":"2026-08-18T22:34:19Z","signature_id":"1436381491ef7a28ce649abbbc1836954d75bfce861a1ca8490342e646d2e9d2","pr_number":6557,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"67620228b260ef66127b0b69960de12fe2eddbeb199df66329f98d7615a3df8a","failure_pattern":"HCP Recovery should recover an HCP cluster with control plane rollback after nodepool and control plane upgrade on 4.20 stable: Timed out after \u003cduration\u003es.","search_query":"Timed out after 900.001s.","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with control plane rollback after nodepool and control plane upgrade on 4.20 stable","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T23:32:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with control plane rollback after nodepool and control plane upgrade on 4.20 stable","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:258]: Timed out after 900.001s.\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:258]: Timed out after 900.001s.\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6613/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090220408950231040","occurred_at":"2026-08-19T23:32:34Z","signature_id":"5b4be62361d18fc5959a8e5233d7b6439eac67487c82050f8a810538bc1b88e3","pr_number":6613,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bf29a540824c1838b936647fffd1b3e436c2538cb5a221801fa7dbd94f0d3b6c","failure_pattern":"HCP Recovery should recover an HCP cluster with nodepool z-stream downgrade reverted by restore on 4.20 stable: Timed out after \u003cduration\u003es.","search_query":"Timed out after 900.000s.","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with nodepool z-stream downgrade reverted by restore on 4.20 stable","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T23:32:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with nodepool z-stream downgrade reverted by restore on 4.20 stable","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:258]: Timed out after 900.000s.\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:258]: Timed out after 900.000s.\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6613/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090220408950231040","occurred_at":"2026-08-19T23:32:34Z","signature_id":"6c98d24c3441b55b9b59099220f661c6b6de4764b8feedcc5f80ea8bd8e2c909","pr_number":6613,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a41887c3af7419c7f9e3647f2a1aff5f2e4dc467b6a5a8d5c25f3d0c7558c548","failure_pattern":"HCP Recovery should recover an HCP cluster with restore from mid-upgrade backup on 4.20 stable: Timed out after \u003cduration\u003es.","search_query":"Timed out after 900.001s.","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with restore from mid-upgrade backup on 4.20 stable","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T23:32:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with restore from mid-upgrade backup on 4.20 stable","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:258]: Timed out after 900.001s.\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:258]: Timed out after 900.001s.\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6613/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090220408950231040","occurred_at":"2026-08-19T23:32:34Z","signature_id":"5b4be62361d18fc5959a8e5233d7b6439eac67487c82050f8a810538bc1b88e3","pr_number":6613,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7db47e556e61c60f515550ddd173fabbc9066fe4dfb5c0a64a6afa95310aa064","failure_pattern":"HCP Recovery should recover an HCP cluster with y-stream upgrade rollback on 4.20 stable: Timed out after \u003cduration\u003es.","search_query":"Timed out after 900.010s.","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with y-stream upgrade rollback on 4.20 stable","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T23:32:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with y-stream upgrade rollback on 4.20 stable","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:258]: Timed out after 900.010s.\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:258]: Timed out after 900.010s.\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6613/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090220408950231040","occurred_at":"2026-08-19T23:32:34Z","signature_id":"db5b8603ae6248568d969b8cf11aa46bed084ef6615bcd15f505703239f03a68","pr_number":6613,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2b0a67e3deae6e622f86a1056f81d35366a09191c38b03a141de005a8d648dc8","failure_pattern":"HCP Recovery should recover an HCP cluster with z-stream upgrade rollback on 4.20 stable: Timed out after \u003cduration\u003es.","search_query":"Timed out after 900.001s.","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with z-stream upgrade rollback on 4.20 stable","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T23:32:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with z-stream upgrade rollback on 4.20 stable","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:258]: Timed out after 900.001s.\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:258]: Timed out after 900.001s.\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6613/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090220408950231040","occurred_at":"2026-08-19T23:32:34Z","signature_id":"5b4be62361d18fc5959a8e5233d7b6439eac67487c82050f8a810538bc1b88e3","pr_number":6613,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"58fc1cbd5bf08c0e9cacc04febc66fa47a6ea8ac19d3acc5b10cdd7a31b22169","failure_pattern":"RevokeCredentialsAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-w9tp85\" in resourcegroup=\"admin-credential-lifecycle-test-xprwv7fr5n4c\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-w9tp85 in resource group admin-credential-lifecycle-test-xprwv7fr5n4c, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-19T19:44:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:253]: failed to revoke admin credentials for cluster \"admin-cred-lifecycle-w9tp85\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009a6ab0\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-w9tp85\" in resourcegroup=\"admin-credential-lifecycle-test-xprwv7fr5n4c\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-w9tp85 in resource group admin-credential-lifecycle-test-xprwv7fr5n4c, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:253]: failed to revoke admin credentials for cluster \"admin-cred-lifecycle-w9tp85\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009a6ab0\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-w9tp85\" in resourcegroup=\"admin-credential-lifecycle-test-xprwv7fr5n4c\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-w9tp85 in resource group admin-credential-lifecycle-test-xprwv7fr5n4c, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6596/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090162950290542592","occurred_at":"2026-08-19T19:44:15Z","signature_id":"ec269b2597fc31d61d2cc332d64658588a04f3f51ea0c63508f0206e3a910359","pr_number":6596,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"923a50890e47da928e11272929c213214b23d4d0575eb0bee74b8d1c72c1cbad","failure_pattern":"Unauthorized","search_query":"Unauthorized","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-10","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-20T09:06:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:231]: Timed out after 300.001s.\nExpected success, but got an error:\n    \u003c*errors.StatusError | 0xc00073de00\u003e: \n    Unauthorized\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:231]: Timed out after 300.001s.\nExpected success, but got an error:\n    \u003c*errors.StatusError | 0xc00073de00\u003e: \n    Unauthorized\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090364783722237952","occurred_at":"2026-08-20T09:06:16Z","signature_id":"31da737492c14a291cad1f6e96261dd134ddc62222103253bc9dfb47dd1c39fc","pr_number":6583,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2992815bad38a5769dd245becb8213e9b3a7e85882b8db6a819eb56049478a21","failure_pattern":"UpdateHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-zrslnbjkrhrx\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-zrslnbjkrhrx, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-18T13:32:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001fcb260\u003e: \n    failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-zrslnbjkrhrx\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-zrslnbjkrhrx, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001fcb260\u003e: \n    failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-zrslnbjkrhrx\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-zrslnbjkrhrx, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6426/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089706907987087360","occurred_at":"2026-08-18T13:32:06Z","signature_id":"0181f75cf8a949240a4d8e98c4bd7dfdd331888277b46eac5309273e5d2ad910","pr_number":6426,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7dd8d2d0eae840e7a1d7a4e2c0c0eff25495785c460c6cce9d24e4e2853ccd76","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e\": EOF","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.fully-private.l5d2.j8125696.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": EOF","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS and private ingress and verify both are only reachable from VNet","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T17:48:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS and private ingress and verify both are only reachable from VNet","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:151]: failed to verify IngressController scope for fully private cluster \"fully-private\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0019a4660\u003e: \n    VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"https://api.fully-private.l5d2.j8125696.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\": EOF\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.fully-private.l5d2.j8125696.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": EOF\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.fully-private.l5d2.j8125696.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": EOF\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private.go:151]: failed to verify IngressController scope for fully private cluster \"fully-private\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0019a4660\u003e: \n    VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"https://api.fully-private.l5d2.j8125696.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\": EOF\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.fully-private.l5d2.j8125696.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": EOF\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.fully-private.l5d2.j8125696.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": EOF\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"805fd9d2b8419ec1131d1fd0e38fac4c9a75781f95e4cfc7ecf5abda4cb48ccb","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"77cea79585bc98e274801940ab9e79052325110f5981c09889832cf18e064369","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: the server was unable to return a response in the time allotted, but may still be processing the request (get apiservices.apiregistration.k8s.io)","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: the server was unable to return a response in the time allotted, but may still be processing the request (get apiservices.apiregistration.k8s.io)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-18T21:37:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:127]: failed to verify cluster health for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00106fef0\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: the server was unable to return a response in the time allotted, but may still be processing the request (get apiservices.apiregistration.k8s.io)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:127]: failed to verify cluster health for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00106fef0\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: the server was unable to return a response in the time allotted, but may still be processing the request (get apiservices.apiregistration.k8s.io)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089829163719462912","occurred_at":"2026-08-18T21:37:54Z","signature_id":"8728f1cf875700934c1adea92929710840f8343d30dc3fc12c339074a5a82fe9","pr_number":6588,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cf087e7d7ad9a23bbaa2e64e6ff30d2ceb433f2c3d39c4dcad0f061219cdfbdf","failure_pattern":"VerifyBasicAccess failed: failed to list services: Get \"\u003curl\u003e","search_query":"VerifyBasicAccess failed: failed to list services: Get \"https://api","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T17:02:48Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:99]: failed to verify HCP cluster \"cilium-cl\" is available\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001057158\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.cilium-cl.vi5q.j7223552.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.cilium-cl.vi5q.j7223552.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.131.82.100:39964-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.cilium-cl.vi5q.j7223552.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.cilium-cl.vi5q.j7223552.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.131.82.100:39964-\u003e172.30.0.10:53: i/o timeout\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:99]: failed to verify HCP cluster \"cilium-cl\" is available\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001057158\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.cilium-cl.vi5q.j7223552.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.cilium-cl.vi5q.j7223552.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.131.82.100:39964-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.cilium-cl.vi5q.j7223552.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.cilium-cl.vi5q.j7223552.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.131.82.100:39964-\u003e172.30.0.10:53: i/o timeout\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090484744067223552","occurred_at":"2026-08-20T17:02:48Z","signature_id":"15ec012c60cd73ddc2be8d5694c34d40f04a7843ebb60a0c6eac3595d1493de1","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1d36f79ec939eea49cb843effa5e1638a7366ebef5983b011f4c93b159cba8c5","failure_pattern":"VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-dj599r namespace are ready: failed to list pods in cilium-connectivity-check-dj599r namespace: client rate limiter Wait returned an error: rate","search_query":"VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-dj599r namespace are ready: failed to list pods in cilium-connectivity-check-dj599r namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-18T13:32:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:198]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0011dced0\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-dj599r namespace are ready: failed to list pods in cilium-connectivity-check-dj599r namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:198]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0011dced0\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-dj599r namespace are ready: failed to list pods in cilium-connectivity-check-dj599r namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6426/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089706907987087360","occurred_at":"2026-08-18T13:32:06Z","signature_id":"6ba5b2f8dc4b317fa5bd8d764492e56158ab18efe72069dca5df470842559e42","pr_number":6426,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"56538debc7c282fbaf42751c4c2a45538d87c5287f64737beff9b071aa52b44a","failure_pattern":"VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"\u003curl\u003e\": EOF","search_query":"VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"https://api.full-priv-kv.c2re.j8125696.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\": EOF","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS, private ingress, and private KeyVault and verify all are operational","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T17:48:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a fully private cluster with private KAS, private ingress, and private KeyVault and verify all are operational","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:182]: failed to verify IngressController scope for fully private cluster with KV \"full-priv-kv\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001c89458\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.full-priv-kv.c2re.j8125696.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": EOF\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.full-priv-kv.c2re.j8125696.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": EOF\n    VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"https://api.full-priv-kv.c2re.j8125696.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\": EOF\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_fully_private_with_kv.go:182]: failed to verify IngressController scope for fully private cluster with KV \"full-priv-kv\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001c89458\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.full-priv-kv.c2re.j8125696.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": EOF\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.full-priv-kv.c2re.j8125696.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": EOF\n    VerifyIngressControllerScope(Internal) failed: failed to get default IngressController: Get \"https://api.full-priv-kv.c2re.j8125696.hcp.osadev.cloud:443/apis/operator.openshift.io/v1/namespaces/openshift-ingress-operator/ingresscontrollers/default\": EOF\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"2c097d2bfcadce600df666a735b77e9a88be50134175d8a10dfc3aac6697ccec","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"21a20c3f2a410873d47baf5997de12bdfb2c72e1a9838fd77eb6baa1c45a7e35","failure_pattern":"VerifyNodesReady failed: can't list nodes in the cluster: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyNodesReady failed: can't list nodes in the cluster: Get \"https://api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud:443/api/v1/nodes\": dial tcp: lookup api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.78.36:33679-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T06:00:01Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:332]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002d88cd8\u003e: \n    VerifyCiliumOperational failed: failed to get cilium namespace: Get \"https://api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud:443/api/v1/namespaces/kube-system\": dial tcp: lookup api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.78.36:33679-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.78.36:33679-\u003e172.30.0.10:53: i/o timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.78.36:33679-\u003e172.30.0.10:53: i/o timeout\n    VerifyNodesReady failed: can't list nodes in the cluster: Get \"https://api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud:443/api/v1/nodes\": dial tcp: lookup api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.78.36:33679-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:332]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002d88cd8\u003e: \n    VerifyCiliumOperational failed: failed to get cilium namespace: Get \"https://api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud:443/api/v1/namespaces/kube-system\": dial tcp: lookup api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.78.36:33679-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.78.36:33679-\u003e172.30.0.10:53: i/o timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.78.36:33679-\u003e172.30.0.10:53: i/o timeout\n    VerifyNodesReady failed: can't list nodes in the cluster: Get \"https://api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud:443/api/v1/nodes\": dial tcp: lookup api.agg-cluster.qwl8.j7822848.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.129.78.36:33679-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6575/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089955526287822848","occurred_at":"2026-08-19T06:00:01Z","signature_id":"cb57fccbabef48381df2877e8173bbe75ee75f2910390c74e6f7c13befe3aa4b","pr_number":6575,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b0f6b13b4a76d7562287bf1face4f5d9fb1b45d4ebf0047290b8a707a05354ee","failure_pattern":"VerifyNodesReady failed: there are not ready nodes: [\u003cnode\u003e]","search_query":"VerifyNodesReady failed: there are not ready nodes: [agg-cluster-agg-np-a-65ddj-wssb4]","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-19T20:53:01Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:332]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002888ee8\u003e: \n    VerifyNodesReady failed: there are not ready nodes: [agg-cluster-agg-np-a-65ddj-wssb4]\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:332]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002888ee8\u003e: \n    VerifyNodesReady failed: there are not ready nodes: [agg-cluster-agg-np-a-65ddj-wssb4]\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090180258316685312","occurred_at":"2026-08-19T20:53:01Z","signature_id":"7f0eeab00948fd70daf7cbbbde40ef16a2d70b7a0858feb589b98705b5a99b5e","pr_number":6614,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7a87df562eaa8a0bfda8c14840c36e7197e35a3b92f22e4f536b1e3d2ed289ff","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-9sj8n.apps.aro.agg-cluster.j2oj.j2980608.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-9sj8n.apps.aro.agg-cluster.j2oj.j2980608.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9sj8n.apps.aro.agg-cluster.j2oj.j2980608.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T15:19:37Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001626078\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-9sj8n.apps.aro.agg-cluster.j2oj.j2980608.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-9sj8n.apps.aro.agg-cluster.j2oj.j2980608.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9sj8n.apps.aro.agg-cluster.j2oj.j2980608.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001626078\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-9sj8n.apps.aro.agg-cluster.j2oj.j2980608.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-9sj8n.apps.aro.agg-cluster.j2oj.j2980608.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-9sj8n.apps.aro.agg-cluster.j2oj.j2980608.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090096353462980608","occurred_at":"2026-08-19T15:19:37Z","signature_id":"67b3ef07ae961ffa750c120255e7902812bbf6a12a0a0d8076d36f33d9a552e8","pr_number":6547,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"745254a66491e8b14022dbd080dc776f7553c02a04c9735ae36d5e5593508f47","failure_pattern":"Yet, Consistently failed because the matcher was not satisfied:","search_query":"Yet, Consistently failed because the matcher was not satisfied:","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T21:45:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:199]: Failed after 300.754s.\nno schedule should execute new backups while paused\nThe function passed to Consistently returned the following error:\n    \u003c*fmt.wrapError | 0xc0012320a0\u003e: \n    failed to send request: Get \"https://admin.j9896192.hcpsvc.osadev.cloud/admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pause-bkp-e2e-qtdq4l5gfbn7/providers/Microsoft.RedHatOpenshift/hcpOpenShiftClusters/pause-bkp-cluster/backupschedules\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)\n    ...\nAt one point, however, the function did return successfully.\nYet, Consistently failed because the matcher was not satisfied:\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:199]: Failed after 300.754s.\nno schedule should execute new backups while paused\nThe function passed to Consistently returned the following error:\n    \u003c*fmt.wrapError | 0xc0012320a0\u003e: \n    failed to send request: Get \"https://admin.j9896192.hcpsvc.osadev.cloud/admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pause-bkp-e2e-qtdq4l5gfbn7/providers/Microsoft.RedHatOpenshift/hcpOpenShiftClusters/pause-bkp-cluster/backupschedules\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)\n    ...\nAt one point, however, the function did return successfully.\nYet, Consistently failed because the matcher was not satisfied:\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6633/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090555886949896192","occurred_at":"2026-08-20T21:45:38Z","signature_id":"8a333d18823bea22f2d701654aebdd23f71e5242d571b9bb55c50b31b08ad8b0","pr_number":6633,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"2c8d18cdb2b7dd33e8c21a8b6a9a4ce7f6faa5fa98e5020989e6c4479385b197","failure_pattern":"Yet, Eventually failed because the matcher was not satisfied:","search_query":"Yet, Eventually failed because the matcher was not satisfied:","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with a ConfigMap on 4.22 stable","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T23:32:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"HCP Recovery should recover an HCP cluster with a ConfigMap on 4.22 stable","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:287]: Told to stop trying after 30.115s.\nrestore permanently failed: lastCondition=, phase=\nAt one point, however, the function did return successfully.\nYet, Eventually failed because the matcher was not satisfied:\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_recovery.go:287]: Told to stop trying after 30.115s.\nrestore permanently failed: lastCondition=, phase=\nAt one point, however, the function did return successfully.\nYet, Eventually failed because the matcher was not satisfied:\nExpected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6613/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090220408950231040","occurred_at":"2026-08-19T23:32:34Z","signature_id":"8eed1d93c78e7ad13c568987723ce9aa239dc333646b033e1b574892fc1aa6d4","pr_number":6613,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9c14268f5937655bde9e381e9f7b23c185ab03a3643c5982db2abac6823bde46","failure_pattern":"alert [hcp] userJourneyKubeApiserverAvailability1h5m fired","search_query":"alert [hcp] userJourneyKubeApiserverAvailability1h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyKubeApiserverAvailability1h5m does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T14:06:51Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyKubeApiserverAvailability1h5m does not fire","occurrences":1}],"full_error_samples":["alert fired 7 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:14:56Z\n  Severity:    Sev3\n  Labels:      _id=\"6051253d-d60f-4b6d-968a-7f03461c4aa0\", alertname=\"userJourneyKubeApiserverAvailability1h5m\", cluster=\"ci01-j0627968-mgmt-1\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", name=\"j0p2h1j4d3t8x5u\", namespace=\"ocm-arohcpci01-2s9n9h5qdf8gb4n07q0k99jcbmc1n5ti\", resource_id=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-skip-minor-gchzdq-ww2xhlmtfnwr/providers/microsoft.redhatopenshift/hcpopenshiftclusters/np-skip-minor-gchzdq\", severity=\"3\", short_window=\"5m\", slo=\"kas-availability\"\n  Description: Resource ID: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-skip-minor-gchzdq-ww2xhlmtfnwr/providers/microsoft.redhatopenshift/hcpopenshiftclusters/np-skip-minor-gchzdq\nManagement Cluster: ci01-j0627968-mgmt-1\nNamespace: ocm-arohcpci01-2s9n9h5qdf8gb4n07q0k99jcbmc1n5ti\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:14:57Z\n  Severity:    Sev3\n  Labels:      _id=\"77131d63-65bc-44c3-962a-dd847abba09a\", alertname=\"userJourneyKubeApiserverAvailability1h5m\", cluster=\"ci01-j0627968-mgmt-1\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", name=\"o3k2b5p1w0y4f7i\", namespace=\"ocm-arohcpci01-2s9n9jldsfibh07akcms9kk0t1s8lje8\", resource_id=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-d9djl4vwvd9d/providers/microsoft.redhatopenshift/hcpopenshiftclusters/np-autoscale-cluster\", severity=\"3\", short_window=\"5m\", slo=\"kas-availability\"\n  Description: Resource ID: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-d9djl4vwvd9d/providers/microsoft.redhatopenshift/hcpopenshiftclusters/np-autoscale-cluster\nManagement Cluster: ci01-j0627968-mgmt-1\nNamespace: ocm-arohcpci01-2s9n9jldsfibh07akcms9kk0t1s8lje8\n\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:14:57Z\n  Severity:    Sev3\n  Labels:      _id=\"20b982ad-cd1d-43f1-b05c-0c221c712ee7\", alertname=\"userJourneyKubeApiserverAvailability1h5m\", cluster=\"ci01-j0627968-mgmt-1\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", name=\"basic-cluster\", namespace=\"ocm-arohcpci01-2s9n9ia6c2epu24n3nrjgvgsrmalvbl6\", resource_id=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-6hk7rf89rgds/providers/microsoft.redhatopenshift/hcpopenshiftclusters/basic-cluster\", severity=\"3\", short_window=\"5m\", slo=\"kas-availability\"\n  Description: Resource ID: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-6hk7rf89rgds/providers/microsoft.redhatopenshift/hcpopenshiftclusters/basic-cluster\nManagement Cluster: ci01-j0627968-mgmt-1\nNamespace: ocm-arohcpci01-2s9n9ia6c2epu24n3nrjgvgsrmalvbl6\n\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:15:57Z\n  Severity:    Sev3\n  Labels:      _id=\"bea7e516-9fcf-42c8-a569-e7972eace539\", alertname=\"userJourneyKubeApiserverAvailability1h5m\", cluster=\"ci01-j0627968-mgmt-2\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", name=\"d3y0h1r9t4i2z7p\", namespace=\"ocm-arohcpci01-2s9n9lf1s41vdkpkfduekbo1vtbqqdr7\", resource_id=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-zstream-upgrade-4-22-8bz42c64rxjn/providers/microsoft.redhatopenshift/hcpopenshiftclusters/cluster-zstream-4-22-n2r9l8\", severity=\"3\", short_window=\"5m\", slo=\"kas-availability\"\n  Description: Resource ID: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-zstream-upgrade-4-22-8bz42c64rxjn/providers/microsoft.redhatopenshift/hcpopenshiftclusters/cluster-zstream-4-22-n2r9l8\nManagement Cluster: ci01-j0627968-mgmt-2\nNamespace: ocm-arohcpci01-2s9n9lf1s41vdkpkfduekbo1vtbqqdr7\n\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:15:57Z\n  Severity:    Sev3\n  Labels:      _id=\"d7014b11-f60b-4daf-aaf4-d29bfef8600a\", alertname=\"userJourneyKubeApiserverAvailability1h5m\", cluster=\"ci01-j0627968-mgmt-2\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", name=\"y7z1l5m6q1x6c2h\", namespace=\"ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8\", resource_id=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-sn968dwgx24q/providers/microsoft.redhatopenshift/hcpopenshiftclusters/basic-hcp-cluster\", severity=\"3\", short_window=\"5m\", slo=\"kas-availability\"\n  Description: Resource ID: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-sn968dwgx24q/providers/microsoft.redhatopenshift/hcpopenshiftclusters/basic-hcp-cluster\nManagement Cluster: ci01-j0627968-mgmt-2\nNamespace: ocm-arohcpci01-2s9na0lgat4eialup75u1revqm7m6gm8\n\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:15:57Z\n  Severity:    Sev3\n  Labels:      _id=\"2fca8da9-1e0a-4528-82bc-24a68640f216\", alertname=\"userJourneyKubeApiserverAvailability1h5m\", cluster=\"ci01-j0627968-mgmt-1\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", name=\"d7l9h4i9a8u5k9s\", namespace=\"ocm-arohcpci01-2s9n9l6npk7dv191ctaf3b1jnodinlcc\", resource_id=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-serialconsole-bootdiag-q8bm22z27sjd/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster-bootdiag\", severity=\"3\", short_window=\"5m\", slo=\"kas-availability\"\n  Description: Resource ID: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-serialconsole-bootdiag-q8bm22z27sjd/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster-bootdiag\nManagement Cluster: ci01-j0627968-mgmt-1\nNamespace: ocm-arohcpci01-2s9n9l6npk7dv191ctaf3b1jnodinlcc\n\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T16:16:57Z\n  Severity:    Sev3\n  Labels:      _id=\"e3abb0c7-5f3e-45fa-a6c2-3d0e7a13cc5b\", alertname=\"userJourneyKubeApiserverAvailability1h5m\", cluster=\"ci01-j0627968-mgmt-2\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0627968/providers/microsoft.monitor/accounts/hcps-j0627968\", name=\"o9i8l3k0v2i2t4i\", namespace=\"ocm-arohcpci01-2s9n9jdesahjtqdjvpc60o8amh8beqdt\", resource_id=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/disabled-image-registry-fbtznfmgpfmh/providers/microsoft.redhatopenshift/hcpopenshiftclusters/disabled-image-registry-hcp-cluster\", severity=\"3\", short_window=\"5m\", slo=\"kas-availability\"\n  Description: Resource ID: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/disabled-image-registry-fbtznfmgpfmh/providers/microsoft.redhatopenshift/hcpopenshiftclusters/disabled-image-registry-hcp-cluster\nManagement Cluster: ci01-j0627968-mgmt-2\nNamespace: ocm-arohcpci01-2s9n9jdesahjtqdjvpc60o8amh8beqdt"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6070/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090078040460627968","occurred_at":"2026-08-19T14:06:51Z","signature_id":"3a2d3dcfe06b6e61a6b799369fb7b2be07c631aba67738face54f5a991bc059b","pr_number":6070,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"41236d715b79de6086589d5e3fbf56742d4eda8ce1f7956cebf10bb4762334a2","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j1256192 fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j1256192 fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j1256192 does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-18T16:36:46Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j1256192 does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-18T17:51:34Z\n  Ended:       2026-08-18T17:59:33Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 70% averaged over a 10-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089753382901256192","occurred_at":"2026-08-18T16:36:46Z","signature_id":"c8a66989bf0343b565fa3da314117aa183d7f0a34150a848cecfbcf5f963353a","pr_number":5900,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0c158f23d05bb2bcb4ce53b2e8ea76abda6e655547218e7a0fe940e3a9592374","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j2704896 fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j2704896 fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j2704896 does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T04:29:50Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j2704896 does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-21T05:47:43Z\n  Ended:       2026-08-21T06:00:37Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 70% averaged over a 10-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090657608682704896","occurred_at":"2026-08-21T04:29:50Z","signature_id":"c8a66989bf0343b565fa3da314117aa183d7f0a34150a848cecfbcf5f963353a","pr_number":6629,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"827b247c67e0c1c1ff223d0571ae6acbe84e136f93ebb1c1235ad187db53fdf1","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j3334144 fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j3334144 fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j3334144 does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T18:10:59Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j3334144 does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-20T19:14:04Z\n  Ended:       2026-08-20T19:16:55Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 70% averaged over a 10-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-20T19:19:35Z\n  Ended:       2026-08-20T19:38:19Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 70% averaged over a 10-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090501897923334144","occurred_at":"2026-08-20T18:10:59Z","signature_id":"3c20c1a005d26cb7f8535a4b8df78c801d2bd82791d2fe759399a81b073277dc","pr_number":6629,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"6c86829d1a8522011a815ca18b30a4e79db08ad0649db90f205c775879437d85","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7204608 fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7204608 fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7204608 does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T20:15:14Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7204608 does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-20T21:11:04Z\n  Ended:       2026-08-20T21:25:19Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 70% averaged over a 10-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090533136747204608","occurred_at":"2026-08-20T20:15:14Z","signature_id":"c8a66989bf0343b565fa3da314117aa183d7f0a34150a848cecfbcf5f963353a","pr_number":6629,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"71141656790e0301084c3e739bec3bf4dcc7697951964de2fcf8b30c55b1c3d1","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7691136 fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7691136 fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7691136 does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T22:09:10Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7691136 does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-20T23:30:29Z\n  Ended:       2026-08-20T23:50:30Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 70% averaged over a 10-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090566949577691136","occurred_at":"2026-08-20T22:09:10Z","signature_id":"c8a66989bf0343b565fa3da314117aa183d7f0a34150a848cecfbcf5f963353a","pr_number":6629,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2cf0526f26a751e656747eaaa8e7450cbcff553172ce32927513b015717916cb","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j9138688 fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j9138688 fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j9138688 does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T09:00:23Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j9138688 does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-21T10:06:46Z\n  Ended:       2026-08-21T10:20:50Z\n  Severity:    Sev3\n  Description: Cosmos DB normalized RU consumption is above 70% averaged over a 10-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090725692609138688","occurred_at":"2026-08-21T09:00:23Z","signature_id":"c8a66989bf0343b565fa3da314117aa183d7f0a34150a848cecfbcf5f963353a","pr_number":6629,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d281f42786cb2a2de5619c7f3d94cdb69aa344781a27c06b8ed771f436cd9a9a","failure_pattern":"alert [infra] Cosmos DB Throttled Requests (429) High - arohcpci01-rp-j6685312 fired","search_query":"alert [infra] Cosmos DB Throttled Requests (429) High - arohcpci01-rp-j6685312 fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Throttled Requests (429) High - arohcpci01-rp-j6685312 does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T20:53:01Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Throttled Requests (429) High - arohcpci01-rp-j6685312 does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-19T21:58:37Z\n  Ended:       2026-08-19T22:05:25Z\n  Severity:    Sev3\n  Description: Cosmos DB returned more than 100 throttled requests (HTTP 429 - request rate too large) summed over a 5-minute window, evaluated every minute. Sustained 429s mean the workload is exceeding provisioned throughput - investigate hot partitions or increase provisioned throughput (RU/s). A low, steady rate of 429s that client SDKs retry transparently can be normal. https://learn.microsoft.com/azure/cosmos-db/troubleshoot-request-rate-too-large"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6614/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090180258316685312","occurred_at":"2026-08-19T20:53:01Z","signature_id":"0766b6645c6206a3fc1a103b5ed7d9fe07fd1625b37249763edfe4af9b817fbb","pr_number":6614,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T16:39:13Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T17:24:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j9953536-svc\", component=\"monitoring-infrastructure\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j9953536-svc has been unreachable for 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090116384229953536","occurred_at":"2026-08-19T16:39:13Z","signature_id":"8f47276792f56fd80d601111472f61f301ea1c8057abf8c04c2a4640a8c202fd","pr_number":6601,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":15,"prior_last_seen_at":"2026-08-21T01:50:52Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T03:16:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j6439552-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.130:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6439552/providers/microsoft.monitor/accounts/services-j6439552\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-6t89j\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T03:16:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j6439552-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.130:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6439552/providers/microsoft.monitor/accounts/services-j6439552\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-6c4c4dfcb-6t89j\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"1532964a3b40e3b2ec3338f5c64978e5eabba2056b68a3c8f4a6cbc6a64ca184","pr_number":6617,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-13T06:28:29Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-08-24T16:19:07Z\n  Ended:       2026-08-24T16:30:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-cxwt8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e03984bb-ef95-4df0-aec9-95c39b0e3c41\"\n  Description: Pod arobit/arobit-forwarder-cxwt8 (fluentbit) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-08-24T16:20:06Z\n  Ended:       2026-08-24T16:33:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9565440-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.73:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9565440/providers/microsoft.monitor/accounts/services-j9565440\", namespace=\"arobit\", pod=\"arobit-forwarder-cxwt8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e03984bb-ef95-4df0-aec9-95c39b0e3c41\"\n  Description: Pod arobit/arobit-forwarder-cxwt8 (fluentbit) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/5958/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091901058069565440","occurred_at":"2026-08-24T14:50:52Z","signature_id":"fe91f4f69f02671271e4cc6e4aaa4f966388d16d9e660f0bf8e6f1f0d5179d16","pr_number":5958,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fb0cbad5198bdae4515e07599d2b1b6ff3ac6cb7da7109069ce8cd804ef1fb4b","failure_pattern":"alert [svc] PrometheusJobUp fired","search_query":"alert [svc] PrometheusJobUp fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert PrometheusJobUp does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-19T16:39:13Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert PrometheusJobUp does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-19T17:14:16Z\n  Severity:    Sev2\n  Labels:      alertname=\"PrometheusJobUp\", cluster=\"ci01-j9953536-svc\", component=\"monitoring-infrastructure\", severity=\"critical\"\n  Description: Prometheus has not been reachable for the past 10 minutes.\nThis may indicate that the Prometheus server is down, unreachable due to network issues, or experiencing a crash loop.\nCheck the status of the Prometheus pods, service endpoints, and network connectivity."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090116384229953536","occurred_at":"2026-08-19T16:39:13Z","signature_id":"8622c4f05e530429c2435ac6bd159aff27f3109548086ec61b7cd9ecf8f87649","pr_number":6601,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"6bfd7287cb264b36e48f23ab625702748569a1f6c62cf922cf2f1ddfc6b7af87","failure_pattern":"alert [svc] UJClusterProvisionErrors6h30m fired","search_query":"alert [svc] UJClusterProvisionErrors6h30m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert UJClusterProvisionErrors6h30m does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":20,"prior_last_seen_at":"2026-08-21T01:50:52Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert UJClusterProvisionErrors6h30m does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-08-21T03:32:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"UJClusterProvisionErrors6h30m\", cluster=\"ci01-j6439552-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6439552/providers/microsoft.monitor/accounts/services-j6439552\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n  Description: More than 30% of cluster create (install) operations are in failed state sustained over 30 minutes, indicating a medium error budget burn (6x) that would exhaust the 95% SLO budget in ~28 hours."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6617/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090617602316439552","occurred_at":"2026-08-21T01:50:52Z","signature_id":"34c0cccba7f7d6217621e9007ff28fd8c26e2b4db86f8de6918886788462ec89","pr_number":6617,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"e650175e12814e322599e325ba41102fb2cdbd85011eb93727df4167243d0b83","failure_pattern":"all 1 initial nodes should be Ready","search_query":"all 1 initial nodes should be Ready","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T11:53:35Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:174]: Timed out after 1200.004s.\nall 1 initial nodes should be Ready\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:170 with:\nexpected 1 nodes, got output: \nExpected\n    \u003c[]string | len:0, cap:0\u003e: ...\nto have length 1\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:174]: Timed out after 1200.004s.\nall 1 initial nodes should be Ready\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:170 with:\nexpected 1 nodes, got output: \nExpected\n    \u003c[]string | len:0, cap:0\u003e: ...\nto have length 1"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090769277882732544","occurred_at":"2026-08-21T11:53:35Z","signature_id":"7dfbb53772571c55a08382817583ca911847760779d9cb56c53bd66f1d25a026","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T23:44:28Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:159]: Timed out after 660.001s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6584/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090585791523721216","occurred_at":"2026-08-20T23:44:28Z","signature_id":"ee111baab05f6b8ee26f353e865437e1bffda7140d9fc154398f141cd776e6f7","pr_number":6584,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"48a64ec1b4b5565d2f7769f9a521ab2dc62d4c19c2f03ba77ff5b2f0a1ca7e6e","failure_pattern":"can't list nodes in the cluster: Get \"\u003curl\u003e","search_query":"can't list nodes in the cluster: Get \"https://10","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-20T17:48:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:153]: failed to verify initial node count of 1\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000bf6980\u003e: \n    can't list nodes in the cluster: Get \"https://10.0.0.5:443/api/v1/nodes\": dial tcp 10.0.0.5:443: i/o timeout\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:153]: failed to verify initial node count of 1\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000bf6980\u003e: \n    can't list nodes in the cluster: Get \"https://10.0.0.5:443/api/v1/nodes\": dial tcp 10.0.0.5:443: i/o timeout\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090496257268125696","occurred_at":"2026-08-20T17:48:41Z","signature_id":"871c1ef42ecb907672fb82bd7b0f6bfc59c5eb5138fb8df2ac817dd02005754a","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0fa4848af6c56529cc6653d1d00cfad5d473fdb5ab0bf0012e9131fe658c7f80","failure_pattern":"error: failed to create secret Post \"\u003curl\u003e\": stream error: stream ID 1; INTERNAL_ERROR; received from peer","search_query":"error: failed to create secret Post \"https://api.v7q7p2x2x2o2v0i.yexb.j6737024.hcp.osadev.cloud:443/api/v1/namespaces/openshift-config/secrets?fieldManager=kubectl-create\u0026fieldValidation=Strict\": stream error: stream ID 1; INTERNAL_ERROR; received from peer","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-18T16:59:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:359]: failed to create console OAuth client secret for external auth via VM\nUnexpected error:\n    \u003c*errors.errorString | 0xc001e30830\u003e: \n    error: failed to create secret Post \"https://api.v7q7p2x2x2o2v0i.yexb.j6737024.hcp.osadev.cloud:443/api/v1/namespaces/openshift-config/secrets?fieldManager=kubectl-create\u0026fieldValidation=Strict\": stream error: stream ID 1; INTERNAL_ERROR; received from peer\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:359]: failed to create console OAuth client secret for external auth via VM\nUnexpected error:\n    \u003c*errors.errorString | 0xc001e30830\u003e: \n    error: failed to create secret Post \"https://api.v7q7p2x2x2o2v0i.yexb.j6737024.hcp.osadev.cloud:443/api/v1/namespaces/openshift-config/secrets?fieldManager=kubectl-create\u0026fieldValidation=Strict\": stream error: stream ID 1; INTERNAL_ERROR; received from peer\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6588/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089759198756737024","occurred_at":"2026-08-18T16:59:53Z","signature_id":"081ff3487f58257eed2aa6dc1919395f02daa71e2053265099d8039256f06f92","pr_number":6588,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9a4507f47f0192d158049af8a07afbd1c7ecccdd86ac7e743f024025d7a1dd1b","failure_pattern":"failed to create HCP cluster \u003ccluster\u003e: failed starting cluster creation \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\": Get \"\u003curl\u003e\": stream error: stream ID 7; CANCEL; received from peer","search_query":"failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-tslqh4hrqks5\": Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-tslqh4hrqks5?api-version=2021-04-01\": stream error: stream ID 7; CANCEL; received from peer","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T02:50:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00111c6c0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-tslqh4hrqks5\": Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-tslqh4hrqks5?api-version=2021-04-01\": stream error: stream ID 7; CANCEL; received from peer\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00111c6c0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-tslqh4hrqks5\": Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-tslqh4hrqks5?api-version=2021-04-01\": stream error: stream ID 7; CANCEL; received from peer\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089907805891858432","occurred_at":"2026-08-19T02:50:24Z","signature_id":"f81f115cfe4ce2fb632cb9ef6f763ee8ec7e8f43bfe7db385ffdc9661d773ea9","pr_number":6583,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8ba895596f7745a323422710773005ff9b3502f5142ff0a77cd51f9567b2d3c7","failure_pattern":"failed to create namespace: Post \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"failed to create namespace: Post \"https://api.n7q6h5o4u0f8p9g.ng5n.j4461568.hcp.osadev.cloud:443/api/v1/namespaces\": dial tcp: lookup api.n7q6h5o4u0f8p9g.ng5n.j4461568.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.131.32.94:57028-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-18T14:47:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-dfhlct\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d65f40\u003e: \n    failed to create namespace: Post \"https://api.n7q6h5o4u0f8p9g.ng5n.j4461568.hcp.osadev.cloud:443/api/v1/namespaces\": dial tcp: lookup api.n7q6h5o4u0f8p9g.ng5n.j4461568.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.131.32.94:57028-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-dfhlct\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d65f40\u003e: \n    failed to create namespace: Post \"https://api.n7q6h5o4u0f8p9g.ng5n.j4461568.hcp.osadev.cloud:443/api/v1/namespaces\": dial tcp: lookup api.n7q6h5o4u0f8p9g.ng5n.j4461568.hcp.osadev.cloud on 172.30.0.10:53: read udp 10.131.32.94:57028-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6564/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725877364461568","occurred_at":"2026-08-18T14:47:25Z","signature_id":"65a0176de16a6dd461f5c7e22a80c4f2eb7795542e223790c22843182826823e","pr_number":6564,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"9bf717bf4d89c90099f10b3895a8491514cc756149fadaaf0d96d0ac944330c3","failure_pattern":"failed to create namespace: rpc error: code = Unavailable desc = error reading from server: read tcp \u003cip\u003e-\u003e\u003cip\u003e: read: connection timed out","search_query":"failed to create namespace: rpc error: code = Unavailable desc = error reading from server: read tcp 10.128.70.64:48864-\u003e10.128.67.62:2379: read: connection timed out","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-18T13:36:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-fd5jxc\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f27c60\u003e: \n    failed to create namespace: rpc error: code = Unavailable desc = error reading from server: read tcp 10.128.70.64:48864-\u003e10.128.67.62:2379: read: connection timed out\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-fd5jxc\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f27c60\u003e: \n    failed to create namespace: rpc error: code = Unavailable desc = error reading from server: read tcp 10.128.70.64:48864-\u003e10.128.67.62:2379: read: connection timed out\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6585/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089708056525934592","occurred_at":"2026-08-18T13:36:10Z","signature_id":"5bd13b46391f948ac206de12c1ba1018d14040908049e44203c414a222381374","pr_number":6585,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"51411f97b0ca6a75b45db7faf3ee1eadf268adaa51892a177659837a8dfa1072","failure_pattern":"failed to locate cilium chart: looks like \"\u003curl\u003e\" is not a valid chart repository or cannot be reached: Get \"\u003curl\u003e\": dial tcp \u003cip\u003e: connect: network is unreachable","search_query":"failed to locate cilium chart: looks like \"https://helm.cilium.io/\" is not a valid chart repository or cannot be reached: Get \"https://helm.cilium.io/index.yaml\": dial tcp [2606:50c0:8003::153]:443: connect: network is unreachable","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-22T18:27:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:310]: failed to install Cilium chart via Helm SDK\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007d28e0\u003e: \n    failed to locate cilium chart: looks like \"https://helm.cilium.io/\" is not a valid chart repository or cannot be reached: Get \"https://helm.cilium.io/index.yaml\": dial tcp [2606:50c0:8003::153]:443: connect: network is unreachable\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:310]: failed to install Cilium chart via Helm SDK\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007d28e0\u003e: \n    failed to locate cilium chart: looks like \"https://helm.cilium.io/\" is not a valid chart repository or cannot be reached: Get \"https://helm.cilium.io/index.yaml\": dial tcp [2606:50c0:8003::153]:443: connect: network is unreachable\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091230843602210816","occurred_at":"2026-08-22T18:27:40Z","signature_id":"8dcc88eb87847bea80d8d24ee0235e83a6c1dc7f3e44fd2468c0d050417f6533","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fcdc8973bc72f1f3eccc8e8fa24d75136d05bdbdaec8efce65b8dbababc7317e","failure_pattern":"failed to lookup subscription ID for \"\u003credacted-id\u003e\": subscription with name \"\u003credacted-id\u003e\" not found","search_query":"failed to lookup subscription ID for \\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\": subscription with name \\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\" not found","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/global/svc-acr-replication","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-22T02:32:25Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/global/svc-acr-replication","occurrences":1}],"full_error_samples":["time=2026-08-22T02:55:00.244Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=global step=svc-acr-replication err=\"failed to lookup subscription ID for \\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\": subscription with name \\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\" not found\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090990445487198208","occurred_at":"2026-08-22T02:32:25Z","signature_id":"4370e90013292c27b98a5f1edf1b48ee6743a527ca936997a98da21b3f0b0d01","pr_number":6607,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"39c1dd6a33b19f003c027e2db72bcada3748569043d1724afd68eb83224cc6b2","failure_pattern":"failed to query public IP echo service: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"failed to query public IP echo service: Get \"https://checkip.amazonaws.com\": dial tcp: lookup checkip.amazonaws.com on 172.30.0.10:53: read udp 10.131.32.94:57061-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-18T14:47:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:154]: failed to resolve test runner public IP\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1b080\u003e: \n    failed to query public IP echo service: Get \"https://checkip.amazonaws.com\": dial tcp: lookup checkip.amazonaws.com on 172.30.0.10:53: read udp 10.131.32.94:57061-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:154]: failed to resolve test runner public IP\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1b080\u003e: \n    failed to query public IP echo service: Get \"https://checkip.amazonaws.com\": dial tcp: lookup checkip.amazonaws.com on 172.30.0.10:53: read udp 10.131.32.94:57061-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6564/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089725877364461568","occurred_at":"2026-08-18T14:47:25Z","signature_id":"99c21af513a9677fd704510d4424243e21704d7f1fcd9b78b187ff9384181d71","pr_number":6564,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f737920d390791319ed23026fad1541242bc52f2cfd4fe78e124e7424e7ee768","failure_pattern":"list nodes (nodePool=\u003cnodepool\u003e): the server was unable to return a response in the time allotted, but may still be processing the request (get nodes)","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-18T13:57:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to capture node release images before upgrade\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001350120\u003e: \n    list nodes (nodePool=npupgrade-4-21): the server was unable to return a response in the time allotted, but may still be processing the request (get nodes)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to capture node release images before upgrade\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001350120\u003e: \n    list nodes (nodePool=npupgrade-4-21): the server was unable to return a response in the time allotted, but may still be processing the request (get nodes)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2089713360550498304","occurred_at":"2026-08-18T13:57:44Z","signature_id":"7695163ae01d851a06f0783c7c1e43efbdbfa4ec2947ac6748bb87f5bc7ccdfe","pr_number":6583,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2ec88ca94a38e80e8fd52c34dd8310e94dbbbdba81fa0e5a803fb5edd3ec27ed","failure_pattern":"priv-np-scale-np-scale-5kns9-rhgcf Ready worker 64m v1.33.13","search_query":"priv-np-scale-np-scale-5kns9-rhgcf Ready worker 64m v1.33.13","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-23T03:43:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:270]: Timed out after 3600.003s.\nall 1 nodes should be Ready after scale down\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:265 with:\nexpected 1 nodes after scale down, got output: priv-np-scale-np-scale-5kns9-7989x   Ready   worker   73m   v1.33.13\npriv-np-scale-np-scale-5kns9-rhgcf   Ready   worker   64m   v1.33.13\nExpected\n    \u003c[]string | len:2, cap:2\u003e: ...\nto have length 1\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:270]: Timed out after 3600.003s.\nall 1 nodes should be Ready after scale down\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:265 with:\nexpected 1 nodes after scale down, got output: priv-np-scale-np-scale-5kns9-7989x   Ready   worker   73m   v1.33.13\npriv-np-scale-np-scale-5kns9-rhgcf   Ready   worker   64m   v1.33.13\nExpected\n    \u003c[]string | len:2, cap:2\u003e: ...\nto have length 1"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091370674797940736","occurred_at":"2026-08-23T03:43:19Z","signature_id":"e7c295a04eccfabf8a50b991c26c147dc75c254f70efb08c9abc192453da473a","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e131b6de8aab0d4d1241bdfbe5183bba73583ecd4d404f6e1056b123e86af48c","failure_pattern":"priv-np-scale-np-scale-mjh2n-txqzj Ready worker 22m v1.33.13","search_query":"priv-np-scale-np-scale-mjh2n-txqzj Ready worker 22m v1.33.13","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-22T18:27:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and successfully scale nodepool replicas","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:267]: Timed out after 1200.003s.\nall 1 nodes should be Ready after scale down\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:263 with:\nexpected 1 nodes after scale down, got output: priv-np-scale-np-scale-mjh2n-k28ns   Ready   worker   30m   v1.33.13\npriv-np-scale-np-scale-mjh2n-txqzj   Ready   worker   22m   v1.33.13\nExpected\n    \u003c[]string | len:2, cap:2\u003e: ...\nto have length 1\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:267]: Timed out after 1200.003s.\nall 1 nodes should be Ready after scale down\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_private_nodepool_scaling.go:263 with:\nexpected 1 nodes after scale down, got output: priv-np-scale-np-scale-mjh2n-k28ns   Ready   worker   30m   v1.33.13\npriv-np-scale-np-scale-mjh2n-txqzj   Ready   worker   22m   v1.33.13\nExpected\n    \u003c[]string | len:2, cap:2\u003e: ...\nto have length 1"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091230843602210816","occurred_at":"2026-08-22T18:27:40Z","signature_id":"9bf5d57efcb7e35f2e7257cf5c373a661b2c0673357fc56c9a7f7dfa748879da","pr_number":6580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d47ff99fd1bcd56f3dd1a2d45fb345dc45ebaade86da8c2155a6cfa662ad0280","failure_pattern":"public ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"public ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.81p9.j7539456.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.81p9.j7539456.hcp.osadev.cloud on 172.30.0.10:53: no such host","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T14:35:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:190]: Timed out after 600.002s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:188 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.81p9.j7539456.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.81p9.j7539456.hcp.osadev.cloud on 172.30.0.10:53: no such host\nUnexpected error:\n    \u003c*url.Error | 0xc002a34a20\u003e: \n    Get \"https://console-openshift-console.apps.aro.private-kas.81p9.j7539456.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.81p9.j7539456.hcp.osadev.cloud on 172.30.0.10:53: no such host\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:190]: Timed out after 600.002s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:188 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.81p9.j7539456.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.81p9.j7539456.hcp.osadev.cloud on 172.30.0.10:53: no such host\nUnexpected error:\n    \u003c*url.Error | 0xc002a34a20\u003e: \n    Get \"https://console-openshift-console.apps.aro.private-kas.81p9.j7539456.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.81p9.j7539456.hcp.osadev.cloud on 172.30.0.10:53: no such host\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2090085264507539456","occurred_at":"2026-08-19T14:35:33Z","signature_id":"d14499ebb4d5737a04844d1f8159573bec53cf37189db5b4f855c279fdd04f6b","pr_number":6607,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5d433008aba23c344781b86d474b8dbf122c3c1ce0be7c5095e73ef5b595140f","failure_pattern":"run command \"e2e-runcommand-1787582391625387873\" failed with exit code 1 on VM \"\u003cvm\u003e\" (output: \"Client Version: v1.36.4 Kustomize Version: v5.8.1\")","search_query":"run command \"e2e-runcommand-1787582391625387873\" failed with exit code 1 on VM \"private-kas-test-vm\" (output: \"Client Version: v1.36.4\\nKustomize Version: v5.8.1\")","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.17035775127768313,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:160]: kubectl version should succeed from VM via private KAS internal LB (output: )\nUnexpected error:\n    \u003c*errors.errorString | 0xc0007fbf60\u003e: \n    run command \"e2e-runcommand-1787582391625387873\" failed with exit code 1 on VM \"private-kas-test-vm\" (output: \"Client Version: v1.36.4\\nKustomize Version: v5.8.1\")\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:160]: kubectl version should succeed from VM via private KAS internal LB (output: )\nUnexpected error:\n    \u003c*errors.errorString | 0xc0007fbf60\u003e: \n    run command \"e2e-runcommand-1787582391625387873\" failed with exit code 1 on VM \"private-kas-test-vm\" (output: \"Client Version: v1.36.4\\nKustomize Version: v5.8.1\")\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/6429/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2091879954055696384","occurred_at":"2026-08-24T13:27:00Z","signature_id":"d079461036286dee8f18df87fbaabb5b38f1fcaa64aea1bbd3a2fe3d558a9ddb","pr_number":6429,"after_last_push_of_merged_pr":false}]}]},{"environment":"int","summary":{"total_runs":31,"failed_runs":13,"raw_occurrences":16,"matched_occurrences":15,"runs_affected":12},"rows":[{"environment":"int","failure_pattern_id":"98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":9.67741935483871,"anchor_occurrences":3,"after_last_push_count":7,"also_in":["PROD","STG"],"trend_counts":[1,1,0,1,0,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-20T16:42:49Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-l8xph5\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f20140\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-trf9f.apps.aro.r8g4l2d8f3g0v2i.b5du.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-l8xph5\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f20140\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-trf9f.apps.aro.r8g4l2d8f3g0v2i.b5du.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000c8e2d0\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-w89wr.apps.aro.cilium-cl.ecqe.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000c8e2d0\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-w89wr.apps.aro.cilium-cl.ecqe.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:196]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0018ea198\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-jf5mq.apps.aro.cilium-cluster.exnb.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:196]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0018ea198\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-jf5mq.apps.aro.cilium-cluster.exnb.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2091735435544891392","occurred_at":"2026-08-24T03:52:45Z","signature_id":"571e7c77690730a14fa42704d42877387a3fe5f21673f5f238b11729008a3c86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2090479680229478400","occurred_at":"2026-08-20T16:42:49Z","signature_id":"dea774d2681150a0cd95cf0e47e56b2fea06f005627625b51ed01c7c9d66a8f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2089828094020947968","occurred_at":"2026-08-18T21:33:39Z","signature_id":"67c67f1508e8b4c5e106485bef9ecc40e2ca93f5f9bd5e6aa4ae684374b5dd8d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"e0eca219c3110341e80c5a3f64ae984723abc1e1208ac11199b51c76f20de3e7","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.451612903225806,"anchor_occurrences":2,"after_last_push_count":3,"also_in":["DEV","PROD","STG"],"trend_counts":[0,2,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-10","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-18T18:37:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes a GPU nodepool in a single cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00098d800\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-txwg4qgs4hsw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/002755ed-c845-4398-a37b-8daa3b7cdc45\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/002755ed-c845-4398-a37b-8daa3b7cdc45\",\n      \"name\": \"002755ed-c845-4398-a37b-8daa3b7cdc45\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T18:50:47.72810349Z\",\n      \"endTime\": \"2026-08-18T19:09:54.253810369Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00098d800\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-txwg4qgs4hsw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/002755ed-c845-4398-a37b-8daa3b7cdc45\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/002755ed-c845-4398-a37b-8daa3b7cdc45\",\n      \"name\": \"002755ed-c845-4398-a37b-8daa3b7cdc45\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T18:50:47.72810349Z\",\n      \"endTime\": \"2026-08-18T19:09:54.253810369Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:131]: failed to create GPU nodepool gpu-np-1 with VM size Standard_NC4as_T4_v3\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a9e9c0\u003e: \n    failed to create NodePool gpu-np-1: failed waiting for nodepool=\"gpu-np-1\" for cluster \"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-6rqw64hz9r29\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df04d9fe-d188-4567-a852-79007382c2f0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df04d9fe-d188-4567-a852-79007382c2f0\",\n      \"name\": \"df04d9fe-d188-4567-a852-79007382c2f0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T13:10:13.902866674Z\",\n      \"endTime\": \"2026-08-18T13:29:15.631640019Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:131]: failed to create GPU nodepool gpu-np-1 with VM size Standard_NC4as_T4_v3\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a9e9c0\u003e: \n    failed to create NodePool gpu-np-1: failed waiting for nodepool=\"gpu-np-1\" for cluster \"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-6rqw64hz9r29\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df04d9fe-d188-4567-a852-79007382c2f0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df04d9fe-d188-4567-a852-79007382c2f0\",\n      \"name\": \"df04d9fe-d188-4567-a852-79007382c2f0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T13:10:13.902866674Z\",\n      \"endTime\": \"2026-08-18T13:29:15.631640019Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2089783838367551488","occurred_at":"2026-08-18T18:37:47Z","signature_id":"910bbdbc57d54c46c77bac5f7a773bb7a5240de5ea49c4347a70f8ea721f1275","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2089682863220854784","occurred_at":"2026-08-18T11:56:33Z","signature_id":"00cbbf2460a0f78e2867edebb759a4cbf8777cb1f20b47ec15fe40c9b511cc87","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"f36a1bbc5e9fcc844fe3fa2cebef1b990c1c4028fdbe37fca0988cbc08f26ddd","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; detail message Another operation is pending for requested object. Operation group '/operations/groups/id/|subscriptions|XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX|resourceGroups|private-keyvault-459rvjqk6tld|privateDnsZones|privatelink.vaultcore.azure.net' already has 1 operations like '/operations/type/UpsertPrivateDnsZone/id/822c0f2c-dbd9-4336-bdd7-54dca0cd3e73_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' queued.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.225806451612903,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T08:43:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:75]: failed to create customer resources with private key vault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dee6c0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-private-kv-cluster-q9q757\" in resourcegroup=\"private-keyvault-459rvjqk6tld\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-keyvault-459rvjqk6tld/providers/Microsoft.Resources/deployments/customer-infra-private-kv-cluster-q9q757/operationStatuses/08584143040338090197\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"code\\\": \\\"Conflict\\\",\\r\\n  \\\"message\\\": \\\"Another operation is pending for requested object. Operation group '/operations/groups/id/|subscriptions|XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX|resourceGroups|private-keyvault-459rvjqk6tld|privateDnsZones|privatelink.vaultcore.azure.net' already has 1 operations like '/operations/type/UpsertPrivateDnsZone/id/822c0f2c-dbd9-4336-bdd7-54dca0cd3e73_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' queued.\\\"\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:75]: failed to create customer resources with private key vault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dee6c0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-private-kv-cluster-q9q757\" in resourcegroup=\"private-keyvault-459rvjqk6tld\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-keyvault-459rvjqk6tld/providers/Microsoft.Resources/deployments/customer-infra-private-kv-cluster-q9q757/operationStatuses/08584143040338090197\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"code\\\": \\\"Conflict\\\",\\r\\n  \\\"message\\\": \\\"Another operation is pending for requested object. Operation group '/operations/groups/id/|subscriptions|XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX|resourceGroups|private-keyvault-459rvjqk6tld|privateDnsZones|privatelink.vaultcore.azure.net' already has 1 operations like '/operations/type/UpsertPrivateDnsZone/id/822c0f2c-dbd9-4336-bdd7-54dca0cd3e73_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' queued.\\\"\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2090721494970667008","occurred_at":"2026-08-21T08:43:42Z","signature_id":"0760817b38be15b0376ec42c964f4fcf6c68007f8f83cfdd5501d0d518b4010f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"19c7731fda7f45e3249a335bc7ba4753eeec988f740bb716b85a25c2a5edd90f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.225806451612903,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD","STG"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T05:35:18Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000729200\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-9m9kd7rcmdds\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83f17bd3-dd33-461c-a6bb-3c1efcddd145\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83f17bd3-dd33-461c-a6bb-3c1efcddd145\",\n      \"name\": \"83f17bd3-dd33-461c-a6bb-3c1efcddd145\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T06:19:56.001094985Z\",\n      \"endTime\": \"2026-08-19T06:38:56.416812667Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000729200\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-9m9kd7rcmdds\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83f17bd3-dd33-461c-a6bb-3c1efcddd145\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83f17bd3-dd33-461c-a6bb-3c1efcddd145\",\n      \"name\": \"83f17bd3-dd33-461c-a6bb-3c1efcddd145\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T06:19:56.001094985Z\",\n      \"endTime\": \"2026-08-19T06:38:56.416812667Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2089949305547788288","occurred_at":"2026-08-19T05:35:18Z","signature_id":"fdf3255b7b121e3a193ebc95f7ddc22d058f11ef8f585db76055ebe3aa3840be","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"13ca13618a2c4b8b7429b3001f7d02e419916c0961928bfe37af66198711655d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.225806451612903,"anchor_occurrences":1,"after_last_push_count":7,"also_in":["DEV","PROD","STG"],"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-03"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-07T23:58:58Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e43900\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fbxfthcgbs8b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fa0acf9-ae9d-4f99-9948-6f023a7edada\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fa0acf9-ae9d-4f99-9948-6f023a7edada\",\n      \"name\": \"7fa0acf9-ae9d-4f99-9948-6f023a7edada\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T10:43:39.73808157Z\",\n      \"endTime\": \"2026-08-24T11:02:45.476245961Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e43900\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fbxfthcgbs8b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fa0acf9-ae9d-4f99-9948-6f023a7edada\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fa0acf9-ae9d-4f99-9948-6f023a7edada\",\n      \"name\": \"7fa0acf9-ae9d-4f99-9948-6f023a7edada\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T10:43:39.73808157Z\",\n      \"endTime\": \"2026-08-24T11:02:45.476245961Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2091834116805758976","occurred_at":"2026-08-24T10:24:52Z","signature_id":"3fa15812aed9d226c2b6bdead73b3a2c80d16497f78ee163ab8cbfa1074904ed","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"0e49f848ca7d69ae73432116f8c3e2d9ecf6b571282ffce3e6703b901225aa88","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.225806451612903,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD","STG"],"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-23T22:07:27Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-kjzq8f with version 4.21.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004b0bc0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-kjzq8f: failed waiting for cluster=\"np-version-upgrade-cluster-kjzq8f\" in resourcegroup=\"rg-np-version-upgrade-kjzq8f-kc56h5fmvk6m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8dadd8ad-eeaa-4dea-bb90-c4f1255a79d0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8dadd8ad-eeaa-4dea-bb90-c4f1255a79d0\",\n      \"name\": \"8dadd8ad-eeaa-4dea-bb90-c4f1255a79d0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-23T23:22:28.180803579Z\",\n      \"endTime\": \"2026-08-23T23:41:32.080303176Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-kjzq8f with version 4.21.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004b0bc0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-kjzq8f: failed waiting for cluster=\"np-version-upgrade-cluster-kjzq8f\" in resourcegroup=\"rg-np-version-upgrade-kjzq8f-kc56h5fmvk6m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8dadd8ad-eeaa-4dea-bb90-c4f1255a79d0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8dadd8ad-eeaa-4dea-bb90-c4f1255a79d0\",\n      \"name\": \"8dadd8ad-eeaa-4dea-bb90-c4f1255a79d0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-23T23:22:28.180803579Z\",\n      \"endTime\": \"2026-08-23T23:41:32.080303176Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2091648540576059392","occurred_at":"2026-08-23T22:07:27Z","signature_id":"0fa8c01ab0585932acfc8df37080798105e378ae78d09a1e92de90a78ac7d573","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"86acf58965dfb7b96713f6a89ff381ba46ffdd44516e1b61638ec23fb893f80b","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.225806451612903,"anchor_occurrences":1,"after_last_push_count":6,"also_in":["DEV","PROD","STG"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-19T19:42:01Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000c26120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-n4cgqv\" in resourcegroup=\"rg-np-version-upgrade-n4cgqv-22qb5ssqb72c\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0f6000cd-5626-4ea1-84d3-e3c4775f5ae6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0f6000cd-5626-4ea1-84d3-e3c4775f5ae6\",\n      \"name\": \"0f6000cd-5626-4ea1-84d3-e3c4775f5ae6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T21:24:56.829945477Z\",\n      \"endTime\": \"2026-08-19T21:48:59.897240723Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9shr1s413r5943057b8kmkq36ndccb still exists (deletion dispatched at 2026-08-19T21:24:57Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000c26120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-n4cgqv\" in resourcegroup=\"rg-np-version-upgrade-n4cgqv-22qb5ssqb72c\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0f6000cd-5626-4ea1-84d3-e3c4775f5ae6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0f6000cd-5626-4ea1-84d3-e3c4775f5ae6\",\n      \"name\": \"0f6000cd-5626-4ea1-84d3-e3c4775f5ae6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T21:24:56.829945477Z\",\n      \"endTime\": \"2026-08-19T21:48:59.897240723Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9shr1s413r5943057b8kmkq36ndccb still exists (deletion dispatched at 2026-08-19T21:24:57Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2090162390078328832","occurred_at":"2026-08-19T19:42:01Z","signature_id":"83ed69b86924c06c334d849269e853976f842aa7586fae9047aa65459f2dd885","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"ef4dd01f86a43829d70263dd0a02bc0d1329f80083910a288015623f9842f040","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.225806451612903,"anchor_occurrences":1,"after_last_push_count":11,"also_in":["DEV","STG"],"trend_counts":[1,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":10,"prior_last_seen_at":"2026-08-18T15:11:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00101a6a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rkhdl8xqhdmz\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0d86f430-60d6-4c0f-b439-c3ac3403f901\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0d86f430-60d6-4c0f-b439-c3ac3403f901\",\n      \"name\": \"0d86f430-60d6-4c0f-b439-c3ac3403f901\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T16:03:01.445081159Z\",\n      \"endTime\": \"2026-08-18T16:27:09.852869865Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s92v1tvs13qsc1h7oqpk49j9bta62qs still exists (deletion dispatched at 2026-08-18T16:03:02Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00101a6a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rkhdl8xqhdmz\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0d86f430-60d6-4c0f-b439-c3ac3403f901\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0d86f430-60d6-4c0f-b439-c3ac3403f901\",\n      \"name\": \"0d86f430-60d6-4c0f-b439-c3ac3403f901\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T16:03:01.445081159Z\",\n      \"endTime\": \"2026-08-18T16:27:09.852869865Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s92v1tvs13qsc1h7oqpk49j9bta62qs still exists (deletion dispatched at 2026-08-18T16:03:02Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2089731977035911168","occurred_at":"2026-08-18T15:11:43Z","signature_id":"3e23cbc74631d15c85d630c1ca30b110fee629980794188ea0bfc4db05b38130","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"69e3ad9bd0c4124153f6e74cb4797f8d44a9ae591984376d6e63dbb2a5d2ca33","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": no upgrade path to update channel \"\u003cocp-channel\u003e\" is currently available for this cluster; a channel appears in the cluster's desired version channels only when an upgrade edge to a release in that channel exists; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.225806451612903,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T03:16:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:174]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-dxvrk4\" to 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000b99590\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-kbc8vjkst6lr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-dxvrk4\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": no upgrade path to update channel \\\"candidate-4.21\\\" is currently available for this cluster; a channel appears in the cluster's desired version channels only when an upgrade edge to a release in that channel exists\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:174]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-dxvrk4\" to 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000b99590\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-kbc8vjkst6lr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-dxvrk4\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": no upgrade path to update channel \\\"candidate-4.21\\\" is currently available for this cluster; a channel appears in the cluster's desired version channels only when an upgrade edge to a release in that channel exists\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2090639026095656960","occurred_at":"2026-08-21T03:16:00Z","signature_id":"1742306de2425841d8b2f90fcaf3ed04ecc8996c35a137df26e79d915760f264","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"6ae8052fe24068dd122c1d2c69a1fea3114083e0220320db69d2f9ee76d7c61d","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.225806451612903,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD","STG"],"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T08:43:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000693590\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20, targetMinor=4.21) failed: clusterversion status.history has no version in target minor \"4.21\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000693590\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20, targetMinor=4.21) failed: clusterversion status.history has no version in target minor \"4.21\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2090721494970667008","occurred_at":"2026-08-21T08:43:42Z","signature_id":"f3bc24e0d223a99b76059ba01e7fa681c39f26ab9d0f4665bfdfdd33c395a153","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"d414eaa781a1a73b840b0ceb0c494d2cfb36d6ce2f01987a892eec84026f3876","failure_pattern":"failed to select node pool \u003cnodepool\u003e version for \u003cocp-channel\u003e channel: Get \"\u003curl\u003e\": net/http: TLS handsh","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": net/http: TLS handsh","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.225806451612903,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": net/http: TLS handshake timeout\nfail [github.com/Azure/ARO-HCP/test/util/framework/deployment_params.go:207]: failed to select node pool install version for candidate-4.20 channel: Get \"https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-4.20\": net/http: TLS handshake timeout"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2091834116805758976","occurred_at":"2026-08-24T10:24:52Z","signature_id":"c4114fce72966a1339b4e9230a4f088e9e9d8e5a8939f917174c276b7d0a2542","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"77fb076d32302f92d929786f345bdad3139a0df750404c0633de671aa620060b","failure_pattern":"x509: certificate signed by unknown authority","search_query":"x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.225806451612903,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["PROD","STG"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-18T15:11:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.000s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.000s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2089731977035911168","occurred_at":"2026-08-18T15:11:43Z","signature_id":"20cd25cfb84069bf0dbeae90ccfea853127cb3d7c1e8d8f537a39512ce31cc19","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"prod","summary":{"total_runs":29,"failed_runs":22,"raw_occurrences":216,"matched_occurrences":216,"runs_affected":22},"rows":[{"environment":"prod","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":53,"runs_affected":6,"failed_runs":6,"run_impact_percent":20.689655172413794,"anchor_occurrences":53,"after_last_push_count":91,"also_in":["DEV","STG"],"trend_counts":[0,36,13,0,0,4,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":29,"prior_last_seen_at":"2026-08-22T23:00:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-21-5t5lsf5rjn4g/cluster-nightly-4-21-xmf9vj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009ad020\u003e: \n    failed to create HCP cluster cluster-nightly-4-21-xmf9vj: failed waiting for cluster=\"cluster-nightly-4-21-xmf9vj\" in resourcegroup=\"rg-nightly-4-21-5t5lsf5rjn4g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f9d217a-1f3c-416d-8de9-636190e4122d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f9d217a-1f3c-416d-8de9-636190e4122d\",\n      \"name\": \"5f9d217a-1f3c-416d-8de9-636190e4122d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:29:39.828760367Z\",\n      \"endTime\": \"2026-08-22T23:48:43.906494075Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-21-5t5lsf5rjn4g/cluster-nightly-4-21-xmf9vj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009ad020\u003e: \n    failed to create HCP cluster cluster-nightly-4-21-xmf9vj: failed waiting for cluster=\"cluster-nightly-4-21-xmf9vj\" in resourcegroup=\"rg-nightly-4-21-5t5lsf5rjn4g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f9d217a-1f3c-416d-8de9-636190e4122d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f9d217a-1f3c-416d-8de9-636190e4122d\",\n      \"name\": \"5f9d217a-1f3c-416d-8de9-636190e4122d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:29:39.828760367Z\",\n      \"endTime\": \"2026-08-22T23:48:43.906494075Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014ce920\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-smph858sdl8w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b54e40a6-a611-4bbe-8757-d1d8cc90bbc9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b54e40a6-a611-4bbe-8757-d1d8cc90bbc9\",\n      \"name\": \"b54e40a6-a611-4bbe-8757-d1d8cc90bbc9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:04:17.915889936Z\",\n      \"endTime\": \"2026-08-22T23:23:21.720195497Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014ce920\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-smph858sdl8w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b54e40a6-a611-4bbe-8757-d1d8cc90bbc9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b54e40a6-a611-4bbe-8757-d1d8cc90bbc9\",\n      \"name\": \"b54e40a6-a611-4bbe-8757-d1d8cc90bbc9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:04:17.915889936Z\",\n      \"endTime\": \"2026-08-22T23:23:21.720195497Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f80e40\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-d2gz4r6tmvpw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c45d03f9-8ce4-4648-8394-8c6d842c5331\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c45d03f9-8ce4-4648-8394-8c6d842c5331\",\n      \"name\": \"c45d03f9-8ce4-4648-8394-8c6d842c5331\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:43:24.031623519Z\",\n      \"endTime\": \"2026-08-23T00:02:33.649737296Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f80e40\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-d2gz4r6tmvpw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c45d03f9-8ce4-4648-8394-8c6d842c5331\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c45d03f9-8ce4-4648-8394-8c6d842c5331\",\n      \"name\": \"c45d03f9-8ce4-4648-8394-8c6d842c5331\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:43:24.031623519Z\",\n      \"endTime\": \"2026-08-23T00:02:33.649737296Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091299562504327168","occurred_at":"2026-08-22T23:00:44Z","signature_id":"07048632dc11c59fdf44740c1ef4e0eb9f45fa439ab3b48e7a52ad336d44f81c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091299562504327168","occurred_at":"2026-08-22T23:00:44Z","signature_id":"1f772bd9d9a81506f67e6f590fc47b7eb94e1511b429f44411c93cafd6da7abf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091299562504327168","occurred_at":"2026-08-22T23:00:44Z","signature_id":"4a1d354c4397ba3bb89eaf00c81e13c1e76974c518f83a48ea93a10d55e1d0fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091299562504327168","occurred_at":"2026-08-22T23:00:44Z","signature_id":"860de345a8938b4d31a30e61d384ad788970489b8c1a04f8d2964c010566a745","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089972004693217280","occurred_at":"2026-08-19T07:04:47Z","signature_id":"de68c7f1eed1daba4792282100d6af1320f2ddc0cb0b268d2b656a8f240b65b1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089901294310395904","occurred_at":"2026-08-19T02:24:31Z","signature_id":"294303ab3f0803f5e3201beb670ce1fbfdde703c5d9b5762ab488911185921af","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089901294310395904","occurred_at":"2026-08-19T02:24:31Z","signature_id":"adad22468ff9be4b330d88b35e123924c23f8c72219649a6db53350d7a9c0969","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"17a5a27eceaf85709d921f2350714ee653956ec3dce32061b6d2f3d140782f28","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"4d6dee469821e05603630a76eb689956f731af3a89f51c6c199dd524171ec578","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"6ecd34deef78788afd184d4cf3964dbb22bf21a40747f667791db8892dff5003","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"8a1aeb73c86777a710aebb6152ebc210f77bc8f4f677ec8fc7969644072061f8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"97ff50bba7aa31c13e4678e9a9632fb3db34bf9078076b8842c458a51f34a45e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"a72d651993eddb1be51bcdf8348aeb0eb0fea781b88df0939ec404816eebfd02","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"c1066041e12c007a97ab6f62a064213658033f1574328a8adae079e950f3c0f4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"d2a799c25d7e109674af43387baa7b553d3c8e1554a4dfaf3d6785f5da0811f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"d45996193f19fe48fdbc1ef49a2cefe2375e2a2228418ba3eff3c51e456a8a08","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"e4327a3940ec731c1cf75bb7184697633b96a1e968205743c14d33e37455381a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2089849852371406848","occurred_at":"2026-08-18T23:00:06Z","signature_id":"d102826482e4113d48d761f1ada8580b117e1cda924848f0d58d14f224e939d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"041b9aa1ac494f6cd235dc87f8abb9dc909f246b5550b00ac98c7d1424dafc2e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"1f6064d2d14feaffcb29468626955fa32b6e538e40b0347fba636434a9557724","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"2eac66dec7bf14f313360d781e4c751ccbc832dde42410f7465995bb3637cf48","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"388c73a966eb87d13a6b0a890fe517cd0bcf8860d3db0b92ead8ce331caeae3a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"397854bc7a038e9695a0397137da004c93983f8a950df479a759107a3cc350b7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"428bfe04f45e754c0eba10a4521f2088d787ae506f8e0e85f64da65da272c01c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"4d7d389c790fedb98339a6ba13dd9bed3a0dc879558b1fce3f525b58c8ccd9c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"5d2b6914bbf3275c2eb151ae6184f18084177afee3c3532d2f9d7eb408616083","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"657f4980e9dd38d28e0a78d4c26ce05a78c94be99a97c65576350f9dce463536","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"77a4a7473abbefc7aee3c5c3abcc075f102ab2427a8015e17241acc2d3755aea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"7f62cba921b6a83fca905836f915fef862a0b42e45993804cc3552e8d89a761d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"840cc8770a62f72b0a0b11dabcb91665780361842cf566493b2b0ce55407b5e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"8571300fa1257a9fd0b7b6629004f547243f1e44efad3bad8be8675bf140ef5e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"892e81ce4ed37e95b8a7d957b47e55ca0ca53d6afdb829e1bb33b9314db93bac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"89ff037e55a8e22e3f123053c4d148a23715c8ac512b27f63f8f3c413b5cd1cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"8a60db110031773d6b9f423ad140d471edaf25342e94d58f7a36dc55a7757a27","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"8b8a269cb68b60ab7b00650b122aaf1dc5cd55f33c5c53de102618c2ef052975","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"9b745fbedc1e2c99fe9d7b36273eb8671aad0decffb4da47ac9c39ad8454571f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"a194a41cb69234873601b5d7d4824b6c2a1ee1b8eb91cb83915bc5efb321ce44","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"a5813c7f8934fbe4f28c74d94229a0640e82cf197a2bdbea11218aad97721552","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"b1a30a76609d5d3d27983a5b861e33690959716545169541d36294596988972d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"bf7dcf565f0f2a2957f82691b221d58b3cefd2a66fb5c69c090b3c18ea8ec7ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"c68652b7b1657b9d5609c0ba46f3f3668286f595f5934a45c115b641d7b93e29","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"c7bce42e4c8d8d606848ff8f7e90c1c27337f7519189a7c383270bbbbd3d8e59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"cae8e36ec0327d10cd7f99fc9c5d43fe776765c292f37c549269d4db4c8af29c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"cb80aa4fa3b3d3f61f84afb895c0c1837aefb9c829282e41ab20ffd0d224d3a4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"d78649e1fd9154ebf96d2faba3860c1d2ac5c534a98bb7b5bc4affa1351089c8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"dfeb539901ecaa07c3030ff6728b368dbca357f9c4c07dc99536c7bd920ef6c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"e51f6cee86aae5c95401f4477176f4a9ad3db5adfdacc22ddcafd067a1e2d245","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"ead94180f2a4c2f340b11f7aeccacdb870b8bf760d47a749bca876e4d31a74b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"ec3208875d30fe5bc87b02fe17e995f2ea04034d671a588bce72dc22039224c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"ed115fada96b83159176296eb2e9fd5e3a32ae4a8db7626547911f0fb618469b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"ee4741f818d06d65a5a54f104f1f4992219c9c418a462448370d6729e0122666","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"eecf676b63e663b955a0f0873b4e5ef24056571303e07100262c0b39f3f2ffed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"fcef4edcff9b374e839d9048c2154c98d5fd846261391a89edd7bab0bd4f43a4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":48,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.896551724137931,"anchor_occurrences":48,"after_last_push_count":110,"also_in":["STG"],"trend_counts":[0,8,40,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-19T08:22:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:182]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-wh4d6d\" with install version 4.20.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f13300\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-wh4d6d: failed waiting for cluster=\"cp-ystream-upgrade-4-21-wh4d6d\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-jxk96xvwxvvc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/15ccb98b-5e2b-47b1-86e9-30fb0729f99b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/15ccb98b-5e2b-47b1-86e9-30fb0729f99b\",\n      \"name\": \"15ccb98b-5e2b-47b1-86e9-30fb0729f99b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T09:16:07.820643421Z\",\n      \"endTime\": \"2026-08-19T09:35:08.723020497Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:182]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-wh4d6d\" with install version 4.20.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f13300\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-wh4d6d: failed waiting for cluster=\"cp-ystream-upgrade-4-21-wh4d6d\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-jxk96xvwxvvc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/15ccb98b-5e2b-47b1-86e9-30fb0729f99b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/15ccb98b-5e2b-47b1-86e9-30fb0729f99b\",\n      \"name\": \"15ccb98b-5e2b-47b1-86e9-30fb0729f99b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T09:16:07.820643421Z\",\n      \"endTime\": \"2026-08-19T09:35:08.723020497Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:593]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f084a0\u003e: \n    failed to create HCP cluster np-downgrade-bb2pdp: failed waiting for cluster=\"np-downgrade-bb2pdp\" in resourcegroup=\"rg-np-downgrade-bb2pdp-78j5hq9mwzxz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b063fb82-4df1-44ba-b542-bd61ed70bc8d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b063fb82-4df1-44ba-b542-bd61ed70bc8d\",\n      \"name\": \"b063fb82-4df1-44ba-b542-bd61ed70bc8d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T09:57:38.906376214Z\",\n      \"endTime\": \"2026-08-19T10:16:40.410448062Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:593]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f084a0\u003e: \n    failed to create HCP cluster np-downgrade-bb2pdp: failed waiting for cluster=\"np-downgrade-bb2pdp\" in resourcegroup=\"rg-np-downgrade-bb2pdp-78j5hq9mwzxz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b063fb82-4df1-44ba-b542-bd61ed70bc8d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b063fb82-4df1-44ba-b542-bd61ed70bc8d\",\n      \"name\": \"b063fb82-4df1-44ba-b542-bd61ed70bc8d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T09:57:38.906376214Z\",\n      \"endTime\": \"2026-08-19T10:16:40.410448062Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-21-gv9hgw\" with version 4.21.28 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d9ea20\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-gv9hgw: failed waiting for cluster=\"cluster-zstream-4-21-gv9hgw\" in resourcegroup=\"rg-zstream-upgrade-4-21-5tt9ll7pp27c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/72a9448e-8164-458c-aad2-98cbc2592898\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/72a9448e-8164-458c-aad2-98cbc2592898\",\n      \"name\": \"72a9448e-8164-458c-aad2-98cbc2592898\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T09:11:52.253118092Z\",\n      \"endTime\": \"2026-08-19T09:30:58.688039675Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-21-gv9hgw\" with version 4.21.28 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d9ea20\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-gv9hgw: failed waiting for cluster=\"cluster-zstream-4-21-gv9hgw\" in resourcegroup=\"rg-zstream-upgrade-4-21-5tt9ll7pp27c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/72a9448e-8164-458c-aad2-98cbc2592898\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/72a9448e-8164-458c-aad2-98cbc2592898\",\n      \"name\": \"72a9448e-8164-458c-aad2-98cbc2592898\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T09:11:52.253118092Z\",\n      \"endTime\": \"2026-08-19T09:30:58.688039675Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"0df77b7d1116b1c04fb388c2383af174d4b6edbaa4d4e5a3744713660e4cd825","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"121ea564f6e09bf6d08a688fb9b47f055a6c5d91e5fd35c38964f9895d0057df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"18c9039bc69cf95c601b8b08c93114e87e827ceec49c97def972c111d9291734","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"24647b076479f96a2634ffafc9740db03c56368574307be170da28d3a936de8f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"251bfa172e1491f88da4c45657ef7300e4fb56b3ae0ea9655004b140adcae167","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"2701a343eb1fd58a3d5797b441c84d009bacbed6965fa9bac1279afda3db8418","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"2ccbe9bb881222347b950b7dd86ea2c3966dbdb29858df2e337e1ca31af5f444","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"2ed0aa0e674602cb7c5b8c192e2ad3415a211fa80d12efb0b021cdf8e984c5cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"308de7cc3c14ec9fc72a6cd8af45a3836968fc6edba5782bc456c6b5a2e9e60a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"374be18ea34adc499786976b50d2dc64d047e18fd5431bb754367b67361b4f99","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"377ba9ed8cd7e2433c5048eeb092062b02298eb7c15fbf9bd952f870987cc137","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"3a77481299db92370e1beffa62cb32edd380dac97e190a53fe9fd522442ad3d1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"3fd30595a87eff5782ce2c7272e3c28a75152243c69c16e99ffb7477c4fb6e38","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"40a4762f8e74398e439cac9f8cd352ab1bf8be139bb7a59cf0495f2afc520f4c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"4ee15af014c611cc0afbf05913562079c6418194a0746c7629baf22e2cf358f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"55b5d1f08bf9103fe0cfbd2c0bde5e44b3dae7d0845a2c07f727c13c2ed01bb0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"5642d58d8b7f01f3ce33ae5c22193c16d2c94d658b647a580e99eeb646b5e207","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"6f6ba678298e0a25a13f71d351d6e5c0fb1eb55d2124e502903fb9cd124063c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"752382c73c4c0ccf5fe99ae75057e4dc04165a0d62c6bfd96e46376bb189e8ab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"774d54b5dde3957244f2491b600d4232c10ba230e6e985998fab3e2de31f0c7a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"7b690261c0fdaf70aefc4e0c1d69295e13b362a2b7899f5851c901bb278dff67","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"80387530c920441616b9e6a753c637d2569df58f0fba8acf80ac68dcac4ddd89","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"90412c7236f5f1b1251aba9404d1630d44bd21548e7ee491d0975760884da1f8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"909047c87a82a5ca0d1c3f8beb70ef026b4c10d4620d27a7fa62d5b013152d2f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"959c55b5753e9b3f9b64e78a7547fcae2646e1a6ade66a7608e02f8e5fd4199c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"99b921f0371a9472fbf21280597188c121a450f3d0a253a45ee3c150a556fca0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"acdd96c0e3a16a7a9f195bb34ead4a7bccfeb4f1e28f74ff353e6516e5a8db21","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"b020a66f78a03802d4210f860b61d90b466c1ebab9b9330375959b3ea3b125df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"b3411c91676226ed4ddc0cba69349300903c5770fda4254a458cc547822b15b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"cb286eb28131d69c054d95d7b3d4393deb3cdba7ec6f2bdce8f6a9fc662ebbe6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"d47910ed53117197469d11f4e78d7397bece9ab04a780d95efb0d35c0c3332c6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"d500f7e2ec150ee3b234a12a903c5d7f46b52dbef3a1576f0d1c22c8e00ad4b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"d8c637988c494a1d61e14bb1a3ffac6a1036745d05dbbaaa70f0b2eded47b391","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"df17467cb7388f00b42f99219d9938f3a5c47ecd51c8e4a52183038b83c64ccd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"df494084b3e8e6856a3071f0cf84ca0d2623b0cfcf0648837f004cfcb22ea6d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"e40cadc23b17461d009accc7c7c5d344bfe4ed6f56fa055f30ddf18580824e71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"ea6d83f9b96725031baa2eaac74488edfa9eb64e7b13aef9ac312df88f744d11","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"f728b1a15e85fd87d77c58b75efc540f29883c7cc8f3857ee12e8efc0f5d1517","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"f92558b5df4deb14668fd647dcfd93e77a702ffb1646fa364db2954ad3f9b2ab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"fb7b2959617f8b36a74b71f6f4b06d420ea039ffc68fad695e362f09c04be255","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"1c6824d7b323fce6d653b2a7ce27d98832d2ddc2ff92872a85e019181c2e8924","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"1f0a19672803d999645185907742387cf6347669800e68ab21450ca003260aa1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"2395cef2fccc79d8e7bbb73689c161c5d875f0f9169e2b9d6e6c1d449be4106d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"3f8a8c981c1a8e22c7c19bf9cf1e5752615c6cb47c4a059c117462d449fc3da7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"461e1a912b7bcea4d8eb3836e752b3387a785782112d0fb1a0858addb3ffdc8d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"560d6bfcb47daf5b5489be029b573902284f2de8a40e5ea2fbcb254c49dfed26","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"6ae87330291c79960e8d92078b1cd730312ecfdb9817b515d823377e9245692b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"f58c68d35bb7143d6ec3c290d1364a705a65880a610832ade77ff181382c7a4a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-22-cxtdzk, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-cxtdzk in resource group rg-candidate-4-22-9rh5s6cz8tvc, error: failed waiting for cluster=\"cluster-candidate-4-22-cxtdzk\" in resourcegroup=\"rg-candidate-4-22-9rh5s6cz8tvc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-cxtdzk in resource group rg-candidate-4-22-9rh5s6cz8tvc, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":27,"runs_affected":15,"failed_runs":15,"run_impact_percent":51.724137931034484,"anchor_occurrences":27,"after_last_push_count":53,"also_in":["DEV","STG"],"trend_counts":[2,5,10,3,2,2,2,3],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":32,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":7}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001f06450\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m4qs88j7zlw8, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-m4qs88j7zlw8\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m4qs88j7zlw8, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001f06450\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m4qs88j7zlw8, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-m4qs88j7zlw8\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-m4qs88j7zlw8, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-c58j4775b54x/cluster-nightly-5-1-rb2jwr should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001a0d800\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-rb2jwr, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-rb2jwr in resource group rg-nightly-5-1-c58j4775b54x, error: failed waiting for cluster=\"cluster-nightly-5-1-rb2jwr\" in resourcegroup=\"rg-nightly-5-1-c58j4775b54x\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-rb2jwr in resource group rg-nightly-5-1-c58j4775b54x, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-c58j4775b54x/cluster-nightly-5-1-rb2jwr should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001a0d800\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-rb2jwr, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-rb2jwr in resource group rg-nightly-5-1-c58j4775b54x, error: failed waiting for cluster=\"cluster-nightly-5-1-rb2jwr\" in resourcegroup=\"rg-nightly-5-1-c58j4775b54x\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-rb2jwr in resource group rg-nightly-5-1-c58j4775b54x, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-mqrq7j46wgxv/cluster-candidate-5-0-gm6hwh should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001400690\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-gm6hwh, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-gm6hwh in resource group rg-candidate-5-0-mqrq7j46wgxv, error: failed waiting for cluster=\"cluster-candidate-5-0-gm6hwh\" in resourcegroup=\"rg-candidate-5-0-mqrq7j46wgxv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-gm6hwh in resource group rg-candidate-5-0-mqrq7j46wgxv, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-mqrq7j46wgxv/cluster-candidate-5-0-gm6hwh should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001400690\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-gm6hwh, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-gm6hwh in resource group rg-candidate-5-0-mqrq7j46wgxv, error: failed waiting for cluster=\"cluster-candidate-5-0-gm6hwh\" in resourcegroup=\"rg-candidate-5-0-mqrq7j46wgxv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-gm6hwh in resource group rg-candidate-5-0-mqrq7j46wgxv, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"48cf5ec4badd8d65b424a299cc4f30c373af95f5d42c09e6725e0ee15b8c1667","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"a75e2b44cb494e9ca286b15ddee1a744c32c76fd89cb2c425050bd89a5961c7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","signature_id":"5fc7819289b1249d9f2299f61167ea9b6a16c06d2102e06da9f3562ee86b0ad0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091661989947379712","occurred_at":"2026-08-23T23:00:54Z","signature_id":"9c9613ddf68cabbd7bf2597c127a096449a4ab4a704124ea9fdfd629dd68d4d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091344896685248512","occurred_at":"2026-08-23T02:00:53Z","signature_id":"4bff708c71d99dabe7a7ae9698b7c41e55c0d1e72df16dff74af2e7f9509686b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091299562504327168","occurred_at":"2026-08-22T23:00:44Z","signature_id":"47702ee742345d64441e042bfdcfa97bab40d3a508721722eba88a7afc082a55","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090982470148165632","occurred_at":"2026-08-22T02:00:44Z","signature_id":"2757e229110f4811519f1ff50a6973559f0450d7596fa852667ff0797de1e8f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090937171266506752","occurred_at":"2026-08-21T23:00:43Z","signature_id":"542d678565bcb4bd14e18783f6cd563900f9c155e36aa7035ac1070cce39883f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090619934282354688","occurred_at":"2026-08-21T02:00:08Z","signature_id":"0b9c27a850dd92839570250cc1ea4c147b359cb2854e15193a15f04b24871364","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090574634771550208","occurred_at":"2026-08-20T23:00:08Z","signature_id":"41a62bc60eab6b5ddcd3bdfb1285a3b6400326d86961ab39390e2f9f677abe8c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090574634771550208","occurred_at":"2026-08-20T23:00:08Z","signature_id":"aacace58e98875c1225bb8a8744e8e8416eb18bc3700b8451e6cabf41bd0ffad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090257540720889856","occurred_at":"2026-08-20T02:00:07Z","signature_id":"d37b69e263fbf930ec5cde42a4763ff2e6dfc059aac58455e214660d9ff281b7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090212357799153664","occurred_at":"2026-08-19T23:00:07Z","signature_id":"48db361fd6cbe53d4b314013e1f67f702ce3583723a51cc22976b7761d1e269a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090212357799153664","occurred_at":"2026-08-19T23:00:07Z","signature_id":"4eb1908867ea617b79ef714731840e3f597cbcd49098bec1a8c9d6b2ce2b7421","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"5c48449131a8d336b23406dd697ca42f090d1dc3c91e24c0f6321967915f23f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"7b07eb09a717b0f9075e8c0dbd5167dceb9caf5a8c49f226b9844c0759771559","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"7e524566c996be43d028e5a4f51cdbd9701e5d3b749912cf8b628227b47bc7c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"d9640b3d4577c59f0336a7cadb9c6f63f512e507dba864340fe337ec82787edb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"e362871c873024a97bfec201061c45353b63e2faf5024b5e599e89ca47286de6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"ec12fc460973c9fa872cf3c9849f1f17d27d6d036db485d805694205ed371d6b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"081f1a38953655e4ae6091a8971cca6c1ebceef583b868a8aac6f4e70f09252c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"3747e3fe8cf830b2dbca441528582abc994eebf4e1accb8b03ec49960570f75a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2089849852371406848","occurred_at":"2026-08-18T23:00:06Z","signature_id":"cc47eb855a607ee9689d0e169d40bdfd378887a3dcba5ef2ab82f3db469c5974","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2089849852371406848","occurred_at":"2026-08-18T23:00:06Z","signature_id":"d7305db7bd27d6f90fe8bd6a0dba4e4fb9466ea5946ec562b845a26d40ddc37c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"a667de98947b6324d1e144a014a7ca47dfafe783ef29a1944cdd8d144610abf1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"ac73ed7a746449db979e0e2261907b0458af3d9b801b03787837f6bcdfb9de11","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"fd3ca4f0abe6b38cf9a8288cea30fe69cd19f4c2b6aeb4087204acf6233efaa7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":9,"runs_affected":5,"failed_runs":5,"run_impact_percent":17.24137931034483,"anchor_occurrences":9,"after_last_push_count":9,"also_in":["DEV","INT","STG"],"trend_counts":[0,0,3,3,1,1,1,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001307698\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-22-094635, targetMinor=4.22.0-0.nightly-multi-2026-08-23-080550) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-23-080550\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001307698\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-22-094635, targetMinor=4.22.0-0.nightly-multi-2026-08-23-080550) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-23-080550\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000e06ca8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-22-094635, targetMinor=4.22.0-0.nightly-multi-2026-08-21-221014) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-21-221014\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000e06ca8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-22-094635, targetMinor=4.22.0-0.nightly-multi-2026-08-21-221014) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-21-221014\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f0d698\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-20-065923, targetMinor=4.22.0-0.nightly-multi-2026-08-20-215447) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-20-215447\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f0d698\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-20-065923, targetMinor=4.22.0-0.nightly-multi-2026-08-20-215447) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-20-215447\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091661989947379712","occurred_at":"2026-08-23T23:00:54Z","signature_id":"8cf3b387c284e6fef4c388bce44fc9a4f747d49436eee0a48d049c94e88389e0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091299562504327168","occurred_at":"2026-08-22T23:00:44Z","signature_id":"a602747a6f8ff912aa52c3897a0845124e7f37239c0849398a58cd662ecde56e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090937171266506752","occurred_at":"2026-08-21T23:00:43Z","signature_id":"c65bf0079bcd2ce58e12f45cf9f37bbe8f148f64187938cc3ba61a042251c88d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090574634771550208","occurred_at":"2026-08-20T23:00:08Z","signature_id":"01e9a66c2f609a10ecc530784123db30dcce788426aa44da969016690f7813c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090574634771550208","occurred_at":"2026-08-20T23:00:08Z","signature_id":"0381d0695962715c36dc0eb5609e6d72fe4109b956e9cc97e338fa4cfb93e786","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090574634771550208","occurred_at":"2026-08-20T23:00:08Z","signature_id":"3bc36dba37a0ed6333bc3c086324611c7fcddcc4f56b00a369cafc8b3309aabe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090212357799153664","occurred_at":"2026-08-19T23:00:07Z","signature_id":"11b4a821d08161a4eb2a13a1638065b8dff2c9c2f3883f32f3c6227b7bb9744d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090212357799153664","occurred_at":"2026-08-19T23:00:07Z","signature_id":"2c525ebc161de7eb9503106b010319d214b7fa6e2c59326d7f45c296b91aa5eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090212357799153664","occurred_at":"2026-08-19T23:00:07Z","signature_id":"931c8963596d2fb1aa67e022746279eb97fffd152acbead1983aa67d0c17f754","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"97958430e3a844b6f5be05213043088dddda4827bc264aa7b71a4bdf2cce51d5","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":6,"runs_affected":4,"failed_runs":4,"run_impact_percent":13.793103448275861,"anchor_occurrences":6,"after_last_push_count":7,"also_in":["DEV","STG"],"trend_counts":[0,0,5,0,0,0,1,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-23T02:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b1f360\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-q76sqr5mpxhb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/365f4568-cb64-4804-914b-9def1df67a45\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/365f4568-cb64-4804-914b-9def1df67a45\",\n      \"name\": \"365f4568-cb64-4804-914b-9def1df67a45\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-23T02:02:43.8249986Z\",\n      \"endTime\": \"2026-08-23T02:21:49.594360516Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b1f360\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-q76sqr5mpxhb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/365f4568-cb64-4804-914b-9def1df67a45\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/365f4568-cb64-4804-914b-9def1df67a45\",\n      \"name\": \"365f4568-cb64-4804-914b-9def1df67a45\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-23T02:02:43.8249986Z\",\n      \"endTime\": \"2026-08-23T02:21:49.594360516Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-22-zg4z5cbh25nm/cluster-candidate-4-22-tbzjpf should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010dc4e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-tbzjpf: failed waiting for cluster=\"cluster-candidate-4-22-tbzjpf\" in resourcegroup=\"rg-candidate-4-22-zg4z5cbh25nm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/52ce422b-7b70-4b90-aa52-fc3fa085301b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/52ce422b-7b70-4b90-aa52-fc3fa085301b\",\n      \"name\": \"52ce422b-7b70-4b90-aa52-fc3fa085301b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:49:18.086270762Z\",\n      \"endTime\": \"2026-08-19T08:08:24.407819096Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-22-zg4z5cbh25nm/cluster-candidate-4-22-tbzjpf should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010dc4e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-tbzjpf: failed waiting for cluster=\"cluster-candidate-4-22-tbzjpf\" in resourcegroup=\"rg-candidate-4-22-zg4z5cbh25nm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/52ce422b-7b70-4b90-aa52-fc3fa085301b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/52ce422b-7b70-4b90-aa52-fc3fa085301b\",\n      \"name\": \"52ce422b-7b70-4b90-aa52-fc3fa085301b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:49:18.086270762Z\",\n      \"endTime\": \"2026-08-19T08:08:24.407819096Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e87320\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-llwvfkrlbl4n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c70527cf-5589-4a4f-a61e-87d227a6cadb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c70527cf-5589-4a4f-a61e-87d227a6cadb\",\n      \"name\": \"c70527cf-5589-4a4f-a61e-87d227a6cadb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:41:07.832237891Z\",\n      \"endTime\": \"2026-08-19T03:00:11.0783774Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e87320\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-llwvfkrlbl4n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c70527cf-5589-4a4f-a61e-87d227a6cadb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c70527cf-5589-4a4f-a61e-87d227a6cadb\",\n      \"name\": \"c70527cf-5589-4a4f-a61e-87d227a6cadb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:41:07.832237891Z\",\n      \"endTime\": \"2026-08-19T03:00:11.0783774Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091344896685248512","occurred_at":"2026-08-23T02:00:53Z","signature_id":"68cb4cfbfaec2dfcb0d787d2c40249532b9f90d9b8bbcba6948bf143a5c89648","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089972004693217280","occurred_at":"2026-08-19T07:04:47Z","signature_id":"8371b215a7c62dde0c1a0f510284a45a0fba9fc6fad21ce69da4e00a46d13002","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089901294310395904","occurred_at":"2026-08-19T02:24:31Z","signature_id":"9287d64e5b4bf5b0b94c17318f08eae05c3e7c9aefa6202c6c1c8578ee69dd22","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"46cb29457af1b267208bce4459af0245262ef4d1ad14c94a0ea2e4ad22bc4095","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"50c252289af604d9dc832800d0b0d6d9e474bee5e4de107b5d0df29194e3c266","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"9cde0add2dcef259d675d7857a2d63c872c0734f52569ee693b1c272cc8cd980","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6411b2b281a97b7e65400b269143638faa6befdec2dd319ac1327622333be049","failure_pattern":"node pool \u003cnodepool\u003e verification failed: \u003cnode\u003e (no version in containerRuntimeVersion \"cri-o://1","search_query":"node pool upgrade verification failed: o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp (no version in containerRuntimeVersion \"cri-o://1","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":17.24137931034483,"anchor_occurrences":5,"after_last_push_count":5,"also_in":["STG"],"trend_counts":[0,0,0,1,1,1,1,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002b4c960\u003e: \n    node pool upgrade verification failed: o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp (no version in containerRuntimeVersion \"cri-o://1.35.4\"); o0h1a3k2n0g4d3b-nps-420-422-9lm97-vs5vw (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"o0h1a3k2n0g4d3b-nps-420-422-9lm97-vs5vw\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]}]\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002b4c960\u003e: \n    node pool upgrade verification failed: o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp (no version in containerRuntimeVersion \"cri-o://1.35.4\"); o0h1a3k2n0g4d3b-nps-420-422-9lm97-vs5vw (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"o0h1a3k2n0g4d3b-nps-420-422-9lm97-vr5xp\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"o0h1a3k2n0g4d3b-nps-420-422-9lm97-vs5vw\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]}]\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000515050\u003e: \n    node pool upgrade verification failed: w1h2v3r9e6e0s7b-nps-420-422-x87c9-8vvqf (no version in containerRuntimeVersion \"cri-o://1.35.4\"); w1h2v3r9e6e0s7b-nps-420-422-x87c9-v4xt9 (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"w1h2v3r9e6e0s7b-nps-420-422-x87c9-8vvqf\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"w1h2v3r9e6e0s7b-nps-420-422-x87c9-v4xt9\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]}]\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000515050\u003e: \n    node pool upgrade verification failed: w1h2v3r9e6e0s7b-nps-420-422-x87c9-8vvqf (no version in containerRuntimeVersion \"cri-o://1.35.4\"); w1h2v3r9e6e0s7b-nps-420-422-x87c9-v4xt9 (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"w1h2v3r9e6e0s7b-nps-420-422-x87c9-8vvqf\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"w1h2v3r9e6e0s7b-nps-420-422-x87c9-v4xt9\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]}]\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000d22130\u003e: \n    node pool upgrade verification failed: w9t9d8g9i9t3j7r-nps-420-422-jxpcn-kdnmh (no version in containerRuntimeVersion \"cri-o://1.35.4\"); w9t9d8g9i9t3j7r-nps-420-422-jxpcn-zmxms (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"w9t9d8g9i9t3j7r-nps-420-422-jxpcn-kdnmh\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b38493011dd1f654b1667e9c131cab5a11344a2ac32f414d457c0444559f12d1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bfa95dcd4a5aa349ba3fc73a0d653779cb753c344a53273cf05b9c32c7055da1\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d8c5e8a02e870bc7d4f63f9d441d972a1de4b77c816cda9252c13e6d604c3678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dbf8d9e6099a2ec5c27e70fc506279c744f9ea26fbe4a601d5dae168e174adf1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b2aeb5f663f8fef2b5de811d6159d00d166c5eba7d4251c98ea5b521ae07f00\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bcb807b6cee3e50ab1dc96db618f84b9d3963dc76507b63fe64db9dde3db5fd2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"w9t9d8g9i9t3j7r-nps-420-422-jxpcn-zmxms\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b38493011dd1f654b1667e9c131cab5a11344a2ac32f414d457c0444559f12d1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bfa95dcd4a5aa349ba3fc73a0d653779cb753c344a53273cf05b9c32c7055da1\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d8c5e8a02e870bc7d4f63f9d441d972a1de4b77c816cda9252c13e6d604c3678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dbf8d9e6099a2ec5c27e70fc506279c744f9ea26fbe4a601d5dae168e174adf1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b2aeb5f663f8fef2b5de811d6159d00d166c5eba7d4251c98ea5b521ae07f00\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bcb807b6cee3e50ab1dc96db618f84b9d3963dc76507b63fe64db9dde3db5fd2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000d22130\u003e: \n    node pool upgrade verification failed: w9t9d8g9i9t3j7r-nps-420-422-jxpcn-kdnmh (no version in containerRuntimeVersion \"cri-o://1.35.4\"); w9t9d8g9i9t3j7r-nps-420-422-jxpcn-zmxms (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"w9t9d8g9i9t3j7r-nps-420-422-jxpcn-kdnmh\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b38493011dd1f654b1667e9c131cab5a11344a2ac32f414d457c0444559f12d1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bfa95dcd4a5aa349ba3fc73a0d653779cb753c344a53273cf05b9c32c7055da1\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d8c5e8a02e870bc7d4f63f9d441d972a1de4b77c816cda9252c13e6d604c3678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dbf8d9e6099a2ec5c27e70fc506279c744f9ea26fbe4a601d5dae168e174adf1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b2aeb5f663f8fef2b5de811d6159d00d166c5eba7d4251c98ea5b521ae07f00\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bcb807b6cee3e50ab1dc96db618f84b9d3963dc76507b63fe64db9dde3db5fd2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"w9t9d8g9i9t3j7r-nps-420-422-jxpcn-zmxms\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b38493011dd1f654b1667e9c131cab5a11344a2ac32f414d457c0444559f12d1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bfa95dcd4a5aa349ba3fc73a0d653779cb753c344a53273cf05b9c32c7055da1\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpprod.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d8c5e8a02e870bc7d4f63f9d441d972a1de4b77c816cda9252c13e6d604c3678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dbf8d9e6099a2ec5c27e70fc506279c744f9ea26fbe4a601d5dae168e174adf1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b2aeb5f663f8fef2b5de811d6159d00d166c5eba7d4251c98ea5b521ae07f00\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bcb807b6cee3e50ab1dc96db618f84b9d3963dc76507b63fe64db9dde3db5fd2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:4eea8f361b3fcb99722b1bf436fc78b999a3ad762bd3162d9566fd6ad156d2f6\",\"arohcpocpprod.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:6e705edac6d68732998d716a353a5481340e396267372d75bf7cb6ee5b10ed45\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"0c1e40a7688ce333e1802f99689a7431268dce71db699919518e6ab02caaba1e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091661989947379712","occurred_at":"2026-08-23T23:00:54Z","signature_id":"e1dc8136aa23907f387f830aa03492aac1f4932daeac05e7f049e4a0362440fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091299562504327168","occurred_at":"2026-08-22T23:00:44Z","signature_id":"31d7531de41f020e51232361d7f51aca32428ae38cb8e072eb49e6914e64342e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090937171266506752","occurred_at":"2026-08-21T23:00:43Z","signature_id":"f9145ac5e2884f8eb9ccef988d1f85c4fe55e8c0ff9c9f95d10d84fcc0da9011","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090574634771550208","occurred_at":"2026-08-20T23:00:08Z","signature_id":"7648865fe6b91dd3936801eb2fb5190babe872afc58659008e03f5db8190bdd7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"06545dfa74499fb7ed3922d254aeb9be3949de04bfba27647afd1af0e1481ab3","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":5,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.896551724137931,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,5,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-19T07:04:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000817120\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9k28vzcm69z8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d038a282-6d26-4349-9d6e-fafe88a9f437\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d038a282-6d26-4349-9d6e-fafe88a9f437\",\n      \"name\": \"d038a282-6d26-4349-9d6e-fafe88a9f437\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:17:50.18783504Z\",\n      \"endTime\": \"2026-08-19T07:36:53.392353745Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000817120\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9k28vzcm69z8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d038a282-6d26-4349-9d6e-fafe88a9f437\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d038a282-6d26-4349-9d6e-fafe88a9f437\",\n      \"name\": \"d038a282-6d26-4349-9d6e-fafe88a9f437\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:17:50.18783504Z\",\n      \"endTime\": \"2026-08-19T07:36:53.392353745Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-21-dvgp2ljncmzv/cluster-candidate-4-21-m56pdm should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000866520\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-m56pdm: failed waiting for cluster=\"cluster-candidate-4-21-m56pdm\" in resourcegroup=\"rg-candidate-4-21-dvgp2ljncmzv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc9b40e4-dff9-45e8-b9ce-61a92321163f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc9b40e4-dff9-45e8-b9ce-61a92321163f\",\n      \"name\": \"dc9b40e4-dff9-45e8-b9ce-61a92321163f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:48:53.852097725Z\",\n      \"endTime\": \"2026-08-19T08:07:54.416992123Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-21-dvgp2ljncmzv/cluster-candidate-4-21-m56pdm should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000866520\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-m56pdm: failed waiting for cluster=\"cluster-candidate-4-21-m56pdm\" in resourcegroup=\"rg-candidate-4-21-dvgp2ljncmzv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc9b40e4-dff9-45e8-b9ce-61a92321163f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc9b40e4-dff9-45e8-b9ce-61a92321163f\",\n      \"name\": \"dc9b40e4-dff9-45e8-b9ce-61a92321163f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:48:53.852097725Z\",\n      \"endTime\": \"2026-08-19T08:07:54.416992123Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f00ec0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-6k6scsrzs89w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/002cf859-3aea-4235-ade3-bfe0bd472945\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/002cf859-3aea-4235-ade3-bfe0bd472945\",\n      \"name\": \"002cf859-3aea-4235-ade3-bfe0bd472945\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:46:18.55837028Z\",\n      \"endTime\": \"2026-08-19T03:05:22.599243963Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f00ec0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-6k6scsrzs89w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/002cf859-3aea-4235-ade3-bfe0bd472945\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/002cf859-3aea-4235-ade3-bfe0bd472945\",\n      \"name\": \"002cf859-3aea-4235-ade3-bfe0bd472945\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:46:18.55837028Z\",\n      \"endTime\": \"2026-08-19T03:05:22.599243963Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089972004693217280","occurred_at":"2026-08-19T07:04:47Z","signature_id":"085089ac4054e8181d0c051218fad6ee05880cc9bf4ac62d6c057d5a72ced7e0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089972004693217280","occurred_at":"2026-08-19T07:04:47Z","signature_id":"4b40e63cf83ed42345aea20290052f9328495d4bd373d02b0972bffda86f67ae","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"02267e9543b2b9550b0b8ec4e4509332326305957c8b1b2cc277d4e59cfe2ac0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"1912b82770035cf45f0662fae4bd7f0a6137993a97ddf66c67b0b086d6cf265f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"c54cac1173c067f540664321c09457b751be9c7b0afb8567c53a6ded23cc7177","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6e0339db7bd54e6d51e0a052774819897d4aae43976b41e9f3679271ea556d4f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-21-184859 to 5.1.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":10.344827586206897,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,1,1,1,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-wzcn8d\" to 5.1.0-0.nightly-multi-2026-08-21-161733\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001250e60\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-wzcn8d\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-6sv96bxs684h\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/28f12e91-44fb-4368-818f-606f837a5513\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/28f12e91-44fb-4368-818f-606f837a5513\",\n      \"name\": \"28f12e91-44fb-4368-818f-606f837a5513\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-23T23:47:34.702003558Z\",\n      \"endTime\": \"2026-08-23T23:47:35.713542837Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-21-184859 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-wzcn8d\" to 5.1.0-0.nightly-multi-2026-08-21-161733\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001250e60\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-wzcn8d\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-6sv96bxs684h\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/28f12e91-44fb-4368-818f-606f837a5513\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/28f12e91-44fb-4368-818f-606f837a5513\",\n      \"name\": \"28f12e91-44fb-4368-818f-606f837a5513\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-23T23:47:34.702003558Z\",\n      \"endTime\": \"2026-08-23T23:47:35.713542837Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-21-184859 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-8pb9fk\" to 5.1.0-0.nightly-multi-2026-08-21-161733\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00162c920\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-8pb9fk\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-rvbnzc42snz2\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/983e4736-d961-45bf-8e63-27df9f4392b0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/983e4736-d961-45bf-8e63-27df9f4392b0\",\n      \"name\": \"983e4736-d961-45bf-8e63-27df9f4392b0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:48:09.603825759Z\",\n      \"endTime\": \"2026-08-22T23:48:09.987134271Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-21-184859 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-8pb9fk\" to 5.1.0-0.nightly-multi-2026-08-21-161733\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00162c920\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-8pb9fk\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-rvbnzc42snz2\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/983e4736-d961-45bf-8e63-27df9f4392b0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/983e4736-d961-45bf-8e63-27df9f4392b0\",\n      \"name\": \"983e4736-d961-45bf-8e63-27df9f4392b0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:48:09.603825759Z\",\n      \"endTime\": \"2026-08-22T23:48:09.987134271Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-21-184859 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-62zzff\" to 5.1.0-0.nightly-multi-2026-08-21-161733\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00102a5c0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-62zzff\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-stlzppcpw4zq\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1b61f43e-2185-4f9f-9b05-3ed88171c66f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1b61f43e-2185-4f9f-9b05-3ed88171c66f\",\n      \"name\": \"1b61f43e-2185-4f9f-9b05-3ed88171c66f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T23:44:09.672524927Z\",\n      \"endTime\": \"2026-08-21T23:44:17.756539054Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-21-184859 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-62zzff\" to 5.1.0-0.nightly-multi-2026-08-21-161733\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00102a5c0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-62zzff\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-stlzppcpw4zq\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1b61f43e-2185-4f9f-9b05-3ed88171c66f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1b61f43e-2185-4f9f-9b05-3ed88171c66f\",\n      \"name\": \"1b61f43e-2185-4f9f-9b05-3ed88171c66f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T23:44:09.672524927Z\",\n      \"endTime\": \"2026-08-21T23:44:17.756539054Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-21-184859 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091661989947379712","occurred_at":"2026-08-23T23:00:54Z","signature_id":"eaa551a9dd1eac38a1c62130fb5c3e51993a843414977ead9e55a0b291b04a09","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091299562504327168","occurred_at":"2026-08-22T23:00:44Z","signature_id":"f374e19a5b7a4b8d52f77129b3b153c431fbc68fe03a795ff29942859e5e3fe4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090937171266506752","occurred_at":"2026-08-21T23:00:43Z","signature_id":"e3efa032fccb6cd516133bd1ad89ebb60e8dae5741cd61caf0ccbc55f84ea58e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"beaf47bd37dc21f47808a0240288ea4322da110d92ca67bf08de0f6793353753","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.20\": must be at least 4.20.35; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":10.344827586206897,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,1,1,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-23T02:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-96fcvx\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008e5080\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-qkzx8x2fwwlv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-96fcvx\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.34\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.35\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-96fcvx\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008e5080\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-qkzx8x2fwwlv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-96fcvx\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.34\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.35\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-n9x84d\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001989aa0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-c8sxt5rwcs2x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-n9x84d\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.34\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.35\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-n9x84d\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001989aa0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-c8sxt5rwcs2x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-n9x84d\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.34\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.35\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-bflg4k\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00170daa0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-5lqcbvhjdxkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-bflg4k\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.34\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.35\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-bflg4k\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00170daa0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-5lqcbvhjdxkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-bflg4k\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.34\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.35\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","signature_id":"67be8c6caed95a193a1745532d025891093937507a496524f899a7d2c80f89f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091344896685248512","occurred_at":"2026-08-23T02:00:53Z","signature_id":"7fadbb50104905453b171581a3b547728bfc635ffabfe3f306a6856ceb68c169","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090982470148165632","occurred_at":"2026-08-22T02:00:44Z","signature_id":"82b312bb71915f291931b82d9a4f4b5450769045c0f3ebb03733360bf9b57bf0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5443585aedc88f5d2589a8fb3795e77266206bcfdb680d3cfbedccaf7bc67f50","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21\": must be at least 4.21.30; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":10.344827586206897,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,1,1,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-23T02:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-znvqfb\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008e8630\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-bb24w6nmgn9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-znvqfb\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.29\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.30\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-znvqfb\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008e8630\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-bb24w6nmgn9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-znvqfb\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.29\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.30\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-jjmbhq\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001168f00\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-zv8kjsp968km/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-jjmbhq\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.29\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.30\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-jjmbhq\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001168f00\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-zv8kjsp968km/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-jjmbhq\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.29\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.30\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-v6tsw8\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000f6bc20\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-wd8hjklvmp49/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-v6tsw8\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.29\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.30\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-v6tsw8\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000f6bc20\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-wd8hjklvmp49/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-v6tsw8\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.29\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.30\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","signature_id":"eabd42ab0d34b7f1d8ca082eb227c6c5a44990af4cb2ad49ebe668cc8663e6ed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091344896685248512","occurred_at":"2026-08-23T02:00:53Z","signature_id":"4a64d043a2a51d28eb3268662ea59869b9f2d4cb03e6c4b5e7e27a8633164231","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090982470148165632","occurred_at":"2026-08-22T02:00:44Z","signature_id":"e1ddce92e347e799f32b65d9f1cb530e81c0d6ecb9fa3b0a4f668a226890dd2e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"2b6de8e3eeb7b1eb9cb9733a09ec9f907952649e2f47b910e6bb29761a829864","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.22\": must be at least 4.22.10; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":10.344827586206897,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,1,1,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-23T02:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-gmlx25\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000703a40\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-b6dm7hck8ksd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-gmlx25\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.10\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.10\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-gmlx25\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000703a40\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-b6dm7hck8ksd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-gmlx25\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.10\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.10\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-7kr5sh\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00155f380\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-85cmmktbjpd6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-7kr5sh\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.10\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.10\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-7kr5sh\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00155f380\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-85cmmktbjpd6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-7kr5sh\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.10\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.10\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-hd26k2\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000914450\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-99xt9th87pkp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-hd26k2\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.10\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.10\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-hd26k2\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000914450\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-99xt9th87pkp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-hd26k2\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.10\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.10\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","signature_id":"233817672ec9f15c61036dd69be2232e5da454215c4ae03262d5d9d1147fe964","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091344896685248512","occurred_at":"2026-08-23T02:00:53Z","signature_id":"e2599145a605851478085e130e5b7fbbe8ead4f7a954be8d2bd8955c7c63ea0b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090982470148165632","occurred_at":"2026-08-22T02:00:44Z","signature_id":"5751bb61a46186e35d46b5f3aa2658a3a2eb89b66543f8c048fbecc5ffa16bc7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6c0df54cac07833305ae54d7a30bcf9a55390b618040466bd5f2843d80e698a8","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":10.344827586206897,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,1,1,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-23T02:00:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000ad6720\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5.0.0-ec.5) failed: install version \"5.0.0-ec.5\" not found in clusterversion/version status.history; cannot confirm the z-stream upgrade started from it\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000ad6720\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5.0.0-ec.5) failed: install version \"5.0.0-ec.5\" not found in clusterversion/version status.history; cannot confirm the z-stream upgrade started from it\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000cde978\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5.0.0-ec.5) failed: install version \"5.0.0-ec.5\" not found in clusterversion/version status.history; cannot confirm the z-stream upgrade started from it\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000cde978\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5.0.0-ec.5) failed: install version \"5.0.0-ec.5\" not found in clusterversion/version status.history; cannot confirm the z-stream upgrade started from it\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001761740\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5.0.0-ec.5) failed: install version \"5.0.0-ec.5\" not found in clusterversion/version status.history; cannot confirm the z-stream upgrade started from it\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001761740\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=5.0.0-ec.5) failed: install version \"5.0.0-ec.5\" not found in clusterversion/version status.history; cannot confirm the z-stream upgrade started from it\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091707289294606336","occurred_at":"2026-08-24T02:00:54Z","signature_id":"3fd31e6b902a75b339639f3cb28f3e152b295a16939ec2f7056ea2da7a8f4e73","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2091344896685248512","occurred_at":"2026-08-23T02:00:53Z","signature_id":"4708ea587292d39020c8eb2261a75c138a9781a6caaa1b7c86a80d70fb0d8b1e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090982470148165632","occurred_at":"2026-08-22T02:00:44Z","signature_id":"7171b853a0dfa100dae5cff129272db18b20b0216e77e4c1ba95edf50cfab602","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"autoscaling-cluster-rm9wbpvfcdwm\", caused by: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rm9wbpvfcdwm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-rm9wbpvfcdwm, error: context deadline exceeded, error: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rm9wbpvfcdwm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-rm9wbpvfcdwm, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":3,"after_last_push_count":4,"also_in":["DEV"],"trend_counts":[0,0,3,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":7,"prior_last_seen_at":"2026-08-19T05:29:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00177ecc0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"pullsecret-test-7k8vgxjbrndr\", caused by: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7k8vgxjbrndr\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-7k8vgxjbrndr, error: context deadline exceeded, error: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7k8vgxjbrndr\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-7k8vgxjbrndr, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00177ecc0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"pullsecret-test-7k8vgxjbrndr\", caused by: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7k8vgxjbrndr\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-7k8vgxjbrndr, error: context deadline exceeded, error: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7k8vgxjbrndr\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-7k8vgxjbrndr, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f46198\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"autoscaling-cluster-rm9wbpvfcdwm\", caused by: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rm9wbpvfcdwm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-rm9wbpvfcdwm, error: context deadline exceeded, error: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rm9wbpvfcdwm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-rm9wbpvfcdwm, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f46198\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"autoscaling-cluster-rm9wbpvfcdwm\", caused by: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rm9wbpvfcdwm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-rm9wbpvfcdwm, error: context deadline exceeded, error: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-rm9wbpvfcdwm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-rm9wbpvfcdwm, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000cdc768\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"oidc-wi-4m2nc64cmj8g\", caused by: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-4m2nc64cmj8g\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-4m2nc64cmj8g, error: context deadline exceeded, error: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-4m2nc64cmj8g\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-4m2nc64cmj8g, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000cdc768\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"oidc-wi-4m2nc64cmj8g\", caused by: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-4m2nc64cmj8g\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-4m2nc64cmj8g, error: context deadline exceeded, error: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-4m2nc64cmj8g\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-4m2nc64cmj8g, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"3df638fe5d43cca0e2f4e149ed492e56b8e1203df35cc7e58fdae4f9e245b884","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"4a223b1d651f299719cd9b6e11edb5f3c3501b11271b66fb68c6008a86d9bda9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"888f561b7f7383f42d7aaa8d1c30b4ffa60e6f2c0e4c3eba34f35a033b8de56c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3d8b21ba20484aaa0ba81955928ba0600e83c5e3854c628e01a8d708233935f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":3,"after_last_push_count":5,"also_in":["STG"],"trend_counts":[0,0,3,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-19T05:29:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000be4420\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-wsd5rv\" in resourcegroup=\"rg-np-skip-minor-wsd5rv-7zc5n446c49n\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/44029b45-2a06-403c-ad77-6b306bc07eac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/44029b45-2a06-403c-ad77-6b306bc07eac\",\n      \"name\": \"44029b45-2a06-403c-ad77-6b306bc07eac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:45:10.619596164Z\",\n      \"endTime\": \"2026-08-19T08:09:15.782431257Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9gasbeacg14154vgqri92i9qj3lj2u still exists (deletion dispatched at 2026-08-19T07:45:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000be4420\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-wsd5rv\" in resourcegroup=\"rg-np-skip-minor-wsd5rv-7zc5n446c49n\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/44029b45-2a06-403c-ad77-6b306bc07eac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/44029b45-2a06-403c-ad77-6b306bc07eac\",\n      \"name\": \"44029b45-2a06-403c-ad77-6b306bc07eac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:45:10.619596164Z\",\n      \"endTime\": \"2026-08-19T08:09:15.782431257Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9gasbeacg14154vgqri92i9qj3lj2u still exists (deletion dispatched at 2026-08-19T07:45:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f06630\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-dg-minor-zm2mgg\" in resourcegroup=\"rg-np-dg-minor-zm2mgg-fwcclg2fsc25\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/655cfcac-fbdc-4612-9646-a9b1d1f8165f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/655cfcac-fbdc-4612-9646-a9b1d1f8165f\",\n      \"name\": \"655cfcac-fbdc-4612-9646-a9b1d1f8165f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:45:55.316555912Z\",\n      \"endTime\": \"2026-08-19T08:09:55.778621843Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9ge0h765335a8886qbg3l13s24maom still exists (deletion dispatched at 2026-08-19T07:45:55Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f06630\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-dg-minor-zm2mgg\" in resourcegroup=\"rg-np-dg-minor-zm2mgg-fwcclg2fsc25\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/655cfcac-fbdc-4612-9646-a9b1d1f8165f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/655cfcac-fbdc-4612-9646-a9b1d1f8165f\",\n      \"name\": \"655cfcac-fbdc-4612-9646-a9b1d1f8165f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:45:55.316555912Z\",\n      \"endTime\": \"2026-08-19T08:09:55.778621843Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9ge0h765335a8886qbg3l13s24maom still exists (deletion dispatched at 2026-08-19T07:45:55Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00116e1b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-dxqx7n\" in resourcegroup=\"rg-np-version-upgrade-dxqx7n-j7h8sl7k4nkw\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6d75e2fa-2f17-4c70-b06f-914c5a7c9ff5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6d75e2fa-2f17-4c70-b06f-914c5a7c9ff5\",\n      \"name\": \"6d75e2fa-2f17-4c70-b06f-914c5a7c9ff5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:31:53.195424705Z\",\n      \"endTime\": \"2026-08-19T07:55:54.488364404Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9g68gmftv667t18n90kkemktfaa0e8 still exists (deletion dispatched at 2026-08-19T07:31:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00116e1b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-dxqx7n\" in resourcegroup=\"rg-np-version-upgrade-dxqx7n-j7h8sl7k4nkw\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6d75e2fa-2f17-4c70-b06f-914c5a7c9ff5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6d75e2fa-2f17-4c70-b06f-914c5a7c9ff5\",\n      \"name\": \"6d75e2fa-2f17-4c70-b06f-914c5a7c9ff5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:31:53.195424705Z\",\n      \"endTime\": \"2026-08-19T07:55:54.488364404Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9g68gmftv667t18n90kkemktfaa0e8 still exists (deletion dispatched at 2026-08-19T07:31:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"88671d66208c0e2cca5412ad058e14717aa66730bf52487d7d3fe6d9e02a93d7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"95e98ccacc154ed064ebee4e99e352acd3e83cfe126b57dd54db13d41e2c9d3b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"e14ae40adcdf76d950a7f4a6d2404b93325e238158216943514be2323a70b355","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.896551724137931,"anchor_occurrences":2,"after_last_push_count":13,"also_in":["DEV","INT","STG"],"trend_counts":[0,0,1,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":20,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:166]: failed to create node pool npupgrade-4-20 with version 4.20.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000706700\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-7ts9cp\" in resourcegroup=\"rg-np-version-upgrade-7ts9cp-bztnq2hvf6sh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6c93bda8-add1-4e48-9eea-0b0a433d2308\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6c93bda8-add1-4e48-9eea-0b0a433d2308\",\n      \"name\": \"6c93bda8-add1-4e48-9eea-0b0a433d2308\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:54:35.924673301Z\",\n      \"endTime\": \"2026-08-21T03:13:35.571507889Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:166]: failed to create node pool npupgrade-4-20 with version 4.20.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000706700\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-7ts9cp\" in resourcegroup=\"rg-np-version-upgrade-7ts9cp-bztnq2hvf6sh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6c93bda8-add1-4e48-9eea-0b0a433d2308\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6c93bda8-add1-4e48-9eea-0b0a433d2308\",\n      \"name\": \"6c93bda8-add1-4e48-9eea-0b0a433d2308\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:54:35.924673301Z\",\n      \"endTime\": \"2026-08-21T03:13:35.571507889Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f5dc80\u003e: \n    failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-sj5szv8t6s8s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37c1aadb-f927-4c30-95d8-892451317321\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37c1aadb-f927-4c30-95d8-892451317321\",\n      \"name\": \"37c1aadb-f927-4c30-95d8-892451317321\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T00:43:05.670558184Z\",\n      \"endTime\": \"2026-08-20T01:02:11.662741691Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f5dc80\u003e: \n    failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-sj5szv8t6s8s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37c1aadb-f927-4c30-95d8-892451317321\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37c1aadb-f927-4c30-95d8-892451317321\",\n      \"name\": \"37c1aadb-f927-4c30-95d8-892451317321\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T00:43:05.670558184Z\",\n      \"endTime\": \"2026-08-20T01:02:11.662741691Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090619934282354688","occurred_at":"2026-08-21T02:00:08Z","signature_id":"53444ab8124f99545024afe8c22d658c5922fe471870c6bfb72c71ce7c7d2fd9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2090221146522783744","occurred_at":"2026-08-19T23:35:30Z","signature_id":"230aa160a5c0cd54b1bd26585190eb1e4882b1ddf6c5769008ac315b46852a77","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"724e7a51f2cd05d0f0c351e24f28344ba4eb40490e6de7ed02ace0e7a7856472","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-23-080550 to 4.23.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.896551724137931,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,1,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-lrdz8v\" to 4.23.0-0.nightly-multi-2026-08-24-110923\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000881340\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-lrdz8v\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-t25q8kvqvdhw\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4393d587-2d85-4674-a4f6-155ceb1138f4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4393d587-2d85-4674-a4f6-155ceb1138f4\",\n      \"name\": \"4393d587-2d85-4674-a4f6-155ceb1138f4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:48:28.850341951Z\",\n      \"endTime\": \"2026-08-24T23:48:30.944709051Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-23-080550 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-lrdz8v\" to 4.23.0-0.nightly-multi-2026-08-24-110923\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000881340\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-lrdz8v\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-t25q8kvqvdhw\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4393d587-2d85-4674-a4f6-155ceb1138f4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4393d587-2d85-4674-a4f6-155ceb1138f4\",\n      \"name\": \"4393d587-2d85-4674-a4f6-155ceb1138f4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:48:28.850341951Z\",\n      \"endTime\": \"2026-08-24T23:48:30.944709051Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-23-080550 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-fld59h\" to 4.23.0-0.nightly-multi-2026-08-22-063717\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004b76c0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-fld59h\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-rjgd8k2jwrfb\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f8fb992a-e4cf-48ed-a1c6-8971aa44e1fa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f8fb992a-e4cf-48ed-a1c6-8971aa44e1fa\",\n      \"name\": \"f8fb992a-e4cf-48ed-a1c6-8971aa44e1fa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-23T23:47:13.066383422Z\",\n      \"endTime\": \"2026-08-23T23:47:20.544658196Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-23-080550 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-fld59h\" to 4.23.0-0.nightly-multi-2026-08-22-063717\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004b76c0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-fld59h\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-rjgd8k2jwrfb\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f8fb992a-e4cf-48ed-a1c6-8971aa44e1fa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f8fb992a-e4cf-48ed-a1c6-8971aa44e1fa\",\n      \"name\": \"f8fb992a-e4cf-48ed-a1c6-8971aa44e1fa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-23T23:47:13.066383422Z\",\n      \"endTime\": \"2026-08-23T23:47:20.544658196Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-23-080550 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"6663bc04df6e96cb243dd9f35fd825b118edab670cd360f1ba5fc79ade7a0f44","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091661989947379712","occurred_at":"2026-08-23T23:00:54Z","signature_id":"3949a84ac57a39a215e47f5d92006980597324fda0c46e302caaa5b9ad11956f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"816d4adf1b0cead0de869e12fb780cc1877997e002f751655ae6980e24a728c2","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.20\": must be at least 4.20.34; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.896551724137931,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["STG"],"trend_counts":[0,0,0,1,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-7kjkxp\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001506ab0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-rncmtdm68xg2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-7kjkxp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.33\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.34\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-7kjkxp\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001506ab0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-rncmtdm68xg2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-7kjkxp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.33\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.34\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-fhgsjh\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000ad93e0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-khb5brts9d48/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-fhgsjh\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.33\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.34\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-fhgsjh\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000ad93e0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-khb5brts9d48/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-fhgsjh\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.33\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.34\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090619934282354688","occurred_at":"2026-08-21T02:00:08Z","signature_id":"6b18799e0c02120baa6b7406c38ed69946bfb8f075f9525de85fb5d61e35a451","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090257540720889856","occurred_at":"2026-08-20T02:00:07Z","signature_id":"11b69fdcc52114ea6507f2ce22ea5cab6804563d71d415697657f4abc51fb821","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4afd6a1ada377507e7e7e6c2a964127b8acdde371be3fb4961b6586bdd4a43de","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21\": must be at least 4.21.29; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.896551724137931,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["STG"],"trend_counts":[0,0,0,1,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-9hq8tp\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000c1fa10\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-dltt2s2c9z4k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-9hq8tp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.28\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.29\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-9hq8tp\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000c1fa10\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-dltt2s2c9z4k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-9hq8tp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.28\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.29\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-4bcwd7\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001318120\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-dcr28mz4j9bb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-4bcwd7\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.28\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.29\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-4bcwd7\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001318120\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-dcr28mz4j9bb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-4bcwd7\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.28\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.29\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090619934282354688","occurred_at":"2026-08-21T02:00:08Z","signature_id":"e2e437dfbe2809e1622aea345f8304dc17ed03f4c4745b55ee016489aa7dc065","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090257540720889856","occurred_at":"2026-08-20T02:00:07Z","signature_id":"e53f4eb7b6705b82dd48b0f763f426c94d8559f03442eb10e9c7be98a65128e5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"41121f687ee586823386c215e67a10fa5622aef7585d34e6556590734bdccec3","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.22\": must be at least 4.22.9; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.896551724137931,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["STG"],"trend_counts":[0,0,0,1,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-d9vwt2\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001576c60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-bmvjhbv2q2z7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-d9vwt2\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.9\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.9\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-d9vwt2\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001576c60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-bmvjhbv2q2z7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-d9vwt2\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.9\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.9\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-cd76bv\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0012a2480\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-5z4mq62hpnnt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-cd76bv\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.9\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.9\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-cd76bv\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0012a2480\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-5z4mq62hpnnt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-cd76bv\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.9\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.9\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090619934282354688","occurred_at":"2026-08-21T02:00:08Z","signature_id":"b0896529db37c90b279c53d6fc422ea9199968c1d2dd406897213a62eda70455","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090257540720889856","occurred_at":"2026-08-20T02:00:07Z","signature_id":"dd299017d2dd3de6a3a73aeae80d69921f52513a63912d73b2ece7bda43e062d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.896551724137931,"anchor_occurrences":2,"after_last_push_count":17,"also_in":["DEV","STG"],"trend_counts":[0,1,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":23,"prior_last_seen_at":"2026-08-19T08:22:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"27a535ba8e7271d53b46c54f95ae18b9dc4982c3092359251c02f6fc6ee4c3de","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"9013e675989bd3312d95cfd426e5318859765876455d6850dc98ee4d503feff7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":3,"also_in":["DEV","STG"],"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-10","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-18T01:17:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dcad60eb8db626db10bfb0812a18e7c392049987a81c4fcbcdc5140e0032dc41","failure_pattern":"Cluster should become ready within '20.000000' minutes","search_query":"Cluster should become ready within '20.000000' minutes","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T08:22:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:163]: Timed out after 1200.003s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:163]: Timed out after 1200.003s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"694e2847365ce62823438330cc701f9592c480ebed5f88d5217b25baa06ae6c6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5fd9534fdae604c74e3a3a42c818683431299c230e99d1c0094cdeba9bca67ef","failure_pattern":"ERROR CODE: Conflict; detail message Cannot update resource while resource is \"provisioning\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T08:22:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:314]: failed to create HCP cluster np-noedge-chjtcz\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fcc1e0\u003e: \n    failed to create HCP cluster np-noedge-chjtcz: failed starting cluster creation \"np-noedge-chjtcz\" in resourcegroup=\"rg-np-noedge-chjtcz-4lgwg5nz5hfm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-chjtcz-4lgwg5nz5hfm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-chjtcz\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot update resource while resource is \\\"provisioning\\\"\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-chjtcz-4lgwg5nz5hfm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-chjtcz\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:314]: failed to create HCP cluster np-noedge-chjtcz\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fcc1e0\u003e: \n    failed to create HCP cluster np-noedge-chjtcz: failed starting cluster creation \"np-noedge-chjtcz\" in resourcegroup=\"rg-np-noedge-chjtcz-4lgwg5nz5hfm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-chjtcz-4lgwg5nz5hfm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-chjtcz\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot update resource while resource is \\\"provisioning\\\"\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-chjtcz-4lgwg5nz5hfm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-chjtcz\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089991394734116864","occurred_at":"2026-08-19T08:22:33Z","signature_id":"6fed47754c392741faea417d75c6c36322e9e06425428e55e56bbee0033699e2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5850eea44ccf20fbe9121c0cac3952957d3152f6db0ef54a778fb6e1a4f24903","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OK","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T05:29:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:79]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a92d60\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-disabled-image-registry-hcp-cluster-n784sz\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-disabled-image-registry-hcp-cluster-n784sz/operationStatuses/08584144855033284603\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"OK\",\n            \"message\": \"\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:79]: failed to create cluster customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a92d60\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-disabled-image-registry-hcp-cluster-n784sz\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-disabled-image-registry-hcp-cluster-n784sz/operationStatuses/08584144855033284603\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"OK\",\n            \"message\": \"\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"002dc1bd12014246737866976762e493ca1bc3b77a01cecd7b6242c9e5c4054e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"0438873facb9dd98e74695c849423f31b775588ec5b0ef7ead10a8182c141d31","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":12,"also_in":["DEV","STG"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":7,"prior_last_seen_at":"2026-08-19T07:04:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00082a120\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-ml5crr8fbhl2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eceba15d-5a1a-4b27-a4e7-b24724b60536\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eceba15d-5a1a-4b27-a4e7-b24724b60536\",\n      \"name\": \"eceba15d-5a1a-4b27-a4e7-b24724b60536\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:18:43.866677598Z\",\n      \"endTime\": \"2026-08-19T07:37:43.126289194Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00082a120\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-ml5crr8fbhl2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eceba15d-5a1a-4b27-a4e7-b24724b60536\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eceba15d-5a1a-4b27-a4e7-b24724b60536\",\n      \"name\": \"eceba15d-5a1a-4b27-a4e7-b24724b60536\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:18:43.866677598Z\",\n      \"endTime\": \"2026-08-19T07:37:43.126289194Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089972004693217280","occurred_at":"2026-08-19T07:04:47Z","signature_id":"acc605509df755de66458b8411f8ea9e7693a79d546c6990432cabd1a2177e62","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"cb8ca43a79863985fe5fa47ef1bbe80d7bc25200d089461928007bb1c0bcd82f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":9,"also_in":["DEV","STG"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-19T02:00:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-5-0-zjc9p9k8fp8k/cluster-candidate-5-0-kz87c7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9e720\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-kz87c7: failed waiting for cluster=\"cluster-candidate-5-0-kz87c7\" in resourcegroup=\"rg-candidate-5-0-zjc9p9k8fp8k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a3081846-fb1d-49f7-9bc5-d276f1e7c8f7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a3081846-fb1d-49f7-9bc5-d276f1e7c8f7\",\n      \"name\": \"a3081846-fb1d-49f7-9bc5-d276f1e7c8f7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:38:11.46423025Z\",\n      \"endTime\": \"2026-08-19T02:57:21.467415333Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-5-0-zjc9p9k8fp8k/cluster-candidate-5-0-kz87c7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9e720\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-kz87c7: failed waiting for cluster=\"cluster-candidate-5-0-kz87c7\" in resourcegroup=\"rg-candidate-5-0-zjc9p9k8fp8k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a3081846-fb1d-49f7-9bc5-d276f1e7c8f7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a3081846-fb1d-49f7-9bc5-d276f1e7c8f7\",\n      \"name\": \"a3081846-fb1d-49f7-9bc5-d276f1e7c8f7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:38:11.46423025Z\",\n      \"endTime\": \"2026-08-19T02:57:21.467415333Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"51e876eeeb423bd303d9c668183a1e9829a3464504c6584d4d08f6a6cd7ae4c6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"df07fa0d010168626c38c9ee8c7f0b6209b1dfc859d2a8f4c53ac9f63886cbbe","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["STG"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T02:00:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-20-fkndtwz8x29s/cluster-candidate-4-20-kpswjg should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c0ef80\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-kpswjg: failed waiting for cluster=\"cluster-candidate-4-20-kpswjg\" in resourcegroup=\"rg-candidate-4-20-fkndtwz8x29s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5ce8c6d6-e8b9-4d76-97a6-d8ba04a44b30\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5ce8c6d6-e8b9-4d76-97a6-d8ba04a44b30\",\n      \"name\": \"5ce8c6d6-e8b9-4d76-97a6-d8ba04a44b30\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:36:44.929925762Z\",\n      \"endTime\": \"2026-08-19T02:55:50.765263478Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-20-fkndtwz8x29s/cluster-candidate-4-20-kpswjg should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c0ef80\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-kpswjg: failed waiting for cluster=\"cluster-candidate-4-20-kpswjg\" in resourcegroup=\"rg-candidate-4-20-fkndtwz8x29s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5ce8c6d6-e8b9-4d76-97a6-d8ba04a44b30\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5ce8c6d6-e8b9-4d76-97a6-d8ba04a44b30\",\n      \"name\": \"5ce8c6d6-e8b9-4d76-97a6-d8ba04a44b30\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:36:44.929925762Z\",\n      \"endTime\": \"2026-08-19T02:55:50.765263478Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"88a3c1d52823c5aea1fc6c80298693b077a17e7246695e45a14c8bda69b5b204","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3148230ac2b952709eb29676f2aa7a717a33aa306cef8cd69c888b564066382e","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","INT","STG"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T04:55:12Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e46720\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-mj9b89zp6qrs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/764a11ed-2f54-474f-bdce-7ac64c3a7a20\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/764a11ed-2f54-474f-bdce-7ac64c3a7a20\",\n      \"name\": \"764a11ed-2f54-474f-bdce-7ac64c3a7a20\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T05:36:20.303691403Z\",\n      \"endTime\": \"2026-08-19T05:55:29.476993829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e46720\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-mj9b89zp6qrs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/764a11ed-2f54-474f-bdce-7ac64c3a7a20\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/764a11ed-2f54-474f-bdce-7ac64c3a7a20\",\n      \"name\": \"764a11ed-2f54-474f-bdce-7ac64c3a7a20\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T05:36:20.303691403Z\",\n      \"endTime\": \"2026-08-19T05:55:29.476993829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089939212420780032","occurred_at":"2026-08-19T04:55:12Z","signature_id":"8e433d3376cdbe8b927b5bd34a27c2e303c2977b118596635ec3afc2173c7b6f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"87342952322b831799ec55685129d31670bbfdbd908032dee3aa610155e46b8d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T07:04:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f896e0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fh79ld4nbj75\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5bea30a8-9570-45db-b908-4c4e07deca9b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5bea30a8-9570-45db-b908-4c4e07deca9b\",\n      \"name\": \"5bea30a8-9570-45db-b908-4c4e07deca9b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:16:30.019566976Z\",\n      \"endTime\": \"2026-08-19T07:35:32.987525861Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f896e0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fh79ld4nbj75\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5bea30a8-9570-45db-b908-4c4e07deca9b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5bea30a8-9570-45db-b908-4c4e07deca9b\",\n      \"name\": \"5bea30a8-9570-45db-b908-4c4e07deca9b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:16:30.019566976Z\",\n      \"endTime\": \"2026-08-19T07:35:32.987525861Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089972004693217280","occurred_at":"2026-08-19T07:04:47Z","signature_id":"93b46a6f12f4806ddb007892005a66398adc1f857b40625a31958a9198b118aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"67e9146bba6a263169b2663e0ece2eff489af41d3e74f4e915f7ce1b74288068","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":12,"also_in":["DEV","INT","STG"],"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001088a60\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-9pxxtrlstzqv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/69f46a0a-74d3-4dcf-bf34-2997e6551fc7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/69f46a0a-74d3-4dcf-bf34-2997e6551fc7\",\n      \"name\": \"69f46a0a-74d3-4dcf-bf34-2997e6551fc7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:06:00.903480575Z\",\n      \"endTime\": \"2026-08-21T02:25:03.149034019Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001088a60\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-9pxxtrlstzqv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/69f46a0a-74d3-4dcf-bf34-2997e6551fc7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/69f46a0a-74d3-4dcf-bf34-2997e6551fc7\",\n      \"name\": \"69f46a0a-74d3-4dcf-bf34-2997e6551fc7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:06:00.903480575Z\",\n      \"endTime\": \"2026-08-21T02:25:03.149034019Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2090619934282354688","occurred_at":"2026-08-21T02:00:08Z","signature_id":"34afd7909da6ce5cf526eb6e72eebbe525ca7ddbcc6a4275c8538eb2c2fc8879","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"79d42a294edfd7eaa19516eb5e39cb2fbbc57c83345983aea3a63ac716c4f9fa","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["DEV","INT","STG"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-19T02:00:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000923f40\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-886kmmqnwq8r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a873ac5e-6930-421b-80c5-b0073727d292\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a873ac5e-6930-421b-80c5-b0073727d292\",\n      \"name\": \"a873ac5e-6930-421b-80c5-b0073727d292\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:03:10.761261438Z\",\n      \"endTime\": \"2026-08-19T02:22:19.897779073Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000923f40\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-886kmmqnwq8r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a873ac5e-6930-421b-80c5-b0073727d292\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a873ac5e-6930-421b-80c5-b0073727d292\",\n      \"name\": \"a873ac5e-6930-421b-80c5-b0073727d292\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:03:10.761261438Z\",\n      \"endTime\": \"2026-08-19T02:22:19.897779073Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"ef1bb3e717af80552bc856ab57b6fe7465a664fd7db1d089b9b6a66971694d25","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6ae125442940bde90789cccefefed6da7645f30b5d4ff58e7803d4ddb14827ec","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T07:04:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00161c860\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-x98pjvvxgzk4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/03e64f83-5bd0-4f0b-819b-4b88633c17cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/03e64f83-5bd0-4f0b-819b-4b88633c17cf\",\n      \"name\": \"03e64f83-5bd0-4f0b-819b-4b88633c17cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:16:18.329159963Z\",\n      \"endTime\": \"2026-08-19T07:35:23.12849681Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00161c860\u003e: \n    failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-x98pjvvxgzk4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/03e64f83-5bd0-4f0b-819b-4b88633c17cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/03e64f83-5bd0-4f0b-819b-4b88633c17cf\",\n      \"name\": \"03e64f83-5bd0-4f0b-819b-4b88633c17cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:16:18.329159963Z\",\n      \"endTime\": \"2026-08-19T07:35:23.12849681Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089972004693217280","occurred_at":"2026-08-19T07:04:47Z","signature_id":"63c75abc2f7d924366fc2bd1819a4197f85221191693b9ed0f020de2cfde66eb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1f1ac3bd2f30331d5cb9d90f9185c916aceed97388936aff1c3323c7c86be085","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T02:24:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00091a9c0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-nzcg89cnmszj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/99a00b91-6d0c-48a1-9fdd-2970d104a1e6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/99a00b91-6d0c-48a1-9fdd-2970d104a1e6\",\n      \"name\": \"99a00b91-6d0c-48a1-9fdd-2970d104a1e6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:42:34.622468609Z\",\n      \"endTime\": \"2026-08-19T03:01:41.644884055Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00091a9c0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-nzcg89cnmszj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/99a00b91-6d0c-48a1-9fdd-2970d104a1e6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/99a00b91-6d0c-48a1-9fdd-2970d104a1e6\",\n      \"name\": \"99a00b91-6d0c-48a1-9fdd-2970d104a1e6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:42:34.622468609Z\",\n      \"endTime\": \"2026-08-19T03:01:41.644884055Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089901294310395904","occurred_at":"2026-08-19T02:24:31Z","signature_id":"2f15a24b405efc2994292058653832d4b3ff19b68c4d62f2c5ef6861a750fb7f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"9f0d2f8bd3c175c3be0acb5a8e8fbae146964e77da19031276e4cea79dffa081","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["DEV","INT","STG"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-19T05:29:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011f0378\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-dw5mg9lsjg9t\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/52d1d0b5-d84a-422f-adff-632e7b15bd86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/52d1d0b5-d84a-422f-adff-632e7b15bd86\",\n      \"name\": \"52d1d0b5-d84a-422f-adff-632e7b15bd86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:21:20.266749938Z\",\n      \"endTime\": \"2026-08-19T07:45:24.43604152Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9g587oc7gqeeckl8k5s1283accvgo6 still exists (deletion dispatched at 2026-08-19T07:21:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011f0378\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-dw5mg9lsjg9t\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/52d1d0b5-d84a-422f-adff-632e7b15bd86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/52d1d0b5-d84a-422f-adff-632e7b15bd86\",\n      \"name\": \"52d1d0b5-d84a-422f-adff-632e7b15bd86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:21:20.266749938Z\",\n      \"endTime\": \"2026-08-19T07:45:24.43604152Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9g587oc7gqeeckl8k5s1283accvgo6 still exists (deletion dispatched at 2026-08-19T07:21:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"e1eee6c793650a7c18d76cf2bccbd457734d3a9610df09f4e0a79b15d0c64711","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fec354a667895e675fed1f76acb6cd3afdc6bf9b42e6371dcffcf0807bbb4b53","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","STG"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T05:29:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000cdc8e8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-85gsqcc2zjlm\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/102c7ad9-cfd7-4fbb-b4af-47ae501c8b59\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/102c7ad9-cfd7-4fbb-b4af-47ae501c8b59\",\n      \"name\": \"102c7ad9-cfd7-4fbb-b4af-47ae501c8b59\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:41:39.807626461Z\",\n      \"endTime\": \"2026-08-19T08:05:45.761898963Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9geiu25jubct6f4b5k5ucmu6fmf5pl still exists (deletion dispatched at 2026-08-19T07:41:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000cdc8e8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-85gsqcc2zjlm\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/102c7ad9-cfd7-4fbb-b4af-47ae501c8b59\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/102c7ad9-cfd7-4fbb-b4af-47ae501c8b59\",\n      \"name\": \"102c7ad9-cfd7-4fbb-b4af-47ae501c8b59\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:41:39.807626461Z\",\n      \"endTime\": \"2026-08-19T08:05:45.761898963Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9geiu25jubct6f4b5k5ucmu6fmf5pl still exists (deletion dispatched at 2026-08-19T07:41:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"6aa10261ddbd41782888a9404c01284ae1dd4848412ef57729612cfd72e777cb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"8c42ccf15938c89725e1c4ff0149e014b02931be5a8b7de4535bfb0022695dd9","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T05:29:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000abefa8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-noedge-9b6mxq\" in resourcegroup=\"rg-np-noedge-9b6mxq-8gg2s6c8ddr4\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fc763824-a7c1-4c84-ad4a-52a04db780c3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fc763824-a7c1-4c84-ad4a-52a04db780c3\",\n      \"name\": \"fc763824-a7c1-4c84-ad4a-52a04db780c3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:11:47.482159856Z\",\n      \"endTime\": \"2026-08-19T07:35:54.369742847Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9g7tdod2lgoa5os2e9n6onp6vt2m2l still exists (deletion dispatched at 2026-08-19T07:11:48Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000abefa8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-noedge-9b6mxq\" in resourcegroup=\"rg-np-noedge-9b6mxq-8gg2s6c8ddr4\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fc763824-a7c1-4c84-ad4a-52a04db780c3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fc763824-a7c1-4c84-ad4a-52a04db780c3\",\n      \"name\": \"fc763824-a7c1-4c84-ad4a-52a04db780c3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T07:11:47.482159856Z\",\n      \"endTime\": \"2026-08-19T07:35:54.369742847Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9g7tdod2lgoa5os2e9n6onp6vt2m2l still exists (deletion dispatched at 2026-08-19T07:11:48Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089947767739781120","occurred_at":"2026-08-19T05:29:11Z","signature_id":"9c9601ec6e6a07ae3c3b84a7f7fd8991aebded251120b054ec5547f0853c5c70","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"220c303bbf477a74273991fde7c9a5f336a0663963edfed72cb63f983e8df885","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-21-140538 to 4.21.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T23:00:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-ltpqhc\" to 4.21.0-0.nightly-multi-2026-08-20-065923\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00071a3e0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-ltpqhc\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-c4ngqczk4ph5\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/80cf611d-a2f7-4a43-b5f4-cb99c877bca2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/80cf611d-a2f7-4a43-b5f4-cb99c877bca2\",\n      \"name\": \"80cf611d-a2f7-4a43-b5f4-cb99c877bca2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T23:42:45.54327206Z\",\n      \"endTime\": \"2026-08-21T23:42:47.666030908Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-21-140538 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-ltpqhc\" to 4.21.0-0.nightly-multi-2026-08-20-065923\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00071a3e0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-ltpqhc\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-c4ngqczk4ph5\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/80cf611d-a2f7-4a43-b5f4-cb99c877bca2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/80cf611d-a2f7-4a43-b5f4-cb99c877bca2\",\n      \"name\": \"80cf611d-a2f7-4a43-b5f4-cb99c877bca2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T23:42:45.54327206Z\",\n      \"endTime\": \"2026-08-21T23:42:47.666030908Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-21-140538 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090937171266506752","occurred_at":"2026-08-21T23:00:43Z","signature_id":"cae38e430d8e80e3a74035572e24eda7a12fa1071db2d9d61da01032369a656b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6357d57cb01557dc5021fba5497a64bf5694fab20b293c17f38284979b87902e","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-22-032601 to 4.21.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-22T23:00:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-ljjplf\" to 4.21.0-0.nightly-multi-2026-08-22-094635\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e00d00\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-ljjplf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-rjbgq4lgq7qt\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/14160601-762a-4228-a29b-548613a97ff8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/14160601-762a-4228-a29b-548613a97ff8\",\n      \"name\": \"14160601-762a-4228-a29b-548613a97ff8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:46:44.80771206Z\",\n      \"endTime\": \"2026-08-22T23:46:53.483788723Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-22-032601 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-ljjplf\" to 4.21.0-0.nightly-multi-2026-08-22-094635\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e00d00\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-ljjplf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-rjbgq4lgq7qt\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/14160601-762a-4228-a29b-548613a97ff8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/14160601-762a-4228-a29b-548613a97ff8\",\n      \"name\": \"14160601-762a-4228-a29b-548613a97ff8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:46:44.80771206Z\",\n      \"endTime\": \"2026-08-22T23:46:53.483788723Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-22-032601 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091299562504327168","occurred_at":"2026-08-22T23:00:44Z","signature_id":"f4249a53ee6a3bebd3274b03d3de9e11f0a0d607d4b225bdb1fc0bac2050d70c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"996c48750505877f54e5f4474f9ab414ba67007e2aa67aff8fc371e2a192e936","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-23-070610 to 4.21.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-wxm9hm\" to 4.21.0-0.nightly-multi-2026-08-22-094635\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a7c6c0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-wxm9hm\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-m4n2k2rcnqcz\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ada4714-891c-4466-827a-ecc32dfa2442\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ada4714-891c-4466-827a-ecc32dfa2442\",\n      \"name\": \"1ada4714-891c-4466-827a-ecc32dfa2442\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-23T23:44:35.120120083Z\",\n      \"endTime\": \"2026-08-23T23:44:40.624916223Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-23-070610 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-wxm9hm\" to 4.21.0-0.nightly-multi-2026-08-22-094635\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a7c6c0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-wxm9hm\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-m4n2k2rcnqcz\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ada4714-891c-4466-827a-ecc32dfa2442\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ada4714-891c-4466-827a-ecc32dfa2442\",\n      \"name\": \"1ada4714-891c-4466-827a-ecc32dfa2442\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-23T23:44:35.120120083Z\",\n      \"endTime\": \"2026-08-23T23:44:40.624916223Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-23-070610 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091661989947379712","occurred_at":"2026-08-23T23:00:54Z","signature_id":"0b2bd00bfed9399346e794c2683e5dec3d1647447257005ac477dccff872ac2c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f966d8a6f1dc892e4c6ee21571e53ba7d8d1ed39b375430e469f3616f54b9b95","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-24-135049 to 4.21.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-7z565t\" to 4.21.0-0.nightly-multi-2026-08-24-143120\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009b5de0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-7z565t\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-98477z6czjf7\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d58c2718-67c9-46eb-afc9-3f15d71ef7a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d58c2718-67c9-46eb-afc9-3f15d71ef7a2\",\n      \"name\": \"d58c2718-67c9-46eb-afc9-3f15d71ef7a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:45:23.595556181Z\",\n      \"endTime\": \"2026-08-24T23:45:30.682388649Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-24-135049 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-7z565t\" to 4.21.0-0.nightly-multi-2026-08-24-143120\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009b5de0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-7z565t\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-98477z6czjf7\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d58c2718-67c9-46eb-afc9-3f15d71ef7a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d58c2718-67c9-46eb-afc9-3f15d71ef7a2\",\n      \"name\": \"d58c2718-67c9-46eb-afc9-3f15d71ef7a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:45:23.595556181Z\",\n      \"endTime\": \"2026-08-24T23:45:30.682388649Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.20.0-0.nightly-multi-2026-08-24-135049 to 4.21.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"15bac440b3d4b1dd68415a557e13f0d806f6cdb816f9edcd9423e02924b01f2e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d8c04d75e2d056f1b8b36668a13ab7045b056c9f740c7013c4882cfcfef5d2d6","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.21.0-0.nightly-multi-2026-08-24-143120 to 4.22.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-cbj2dm\" to 4.22.0-0.nightly-multi-2026-08-23-080550\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000df2720\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-cbj2dm\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4ls4ksxqchgg\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\",\n      \"name\": \"d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:48:14.416516082Z\",\n      \"endTime\": \"2026-08-24T23:48:20.787265431Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.21.0-0.nightly-multi-2026-08-24-143120 to 4.22.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-cbj2dm\" to 4.22.0-0.nightly-multi-2026-08-23-080550\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000df2720\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-cbj2dm\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4ls4ksxqchgg\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\",\n      \"name\": \"d1a50e80-0d7c-4afa-9fa8-42a36f49b9c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:48:14.416516082Z\",\n      \"endTime\": \"2026-08-24T23:48:20.787265431Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.21.0-0.nightly-multi-2026-08-24-143120 to 4.22.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"498a13bc157b497dedc0c407f7c032fb09b28d4110c6fbee95fdae33ff5fe349","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"8de5e14024f45fb18338f1663f4f58e82e3a8afdfe682bc869aa5a0214bf4fc2","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-17-010411 to 4.23.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["STG"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T23:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:174]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-d5pz9g\" to 4.23.0-0.nightly-multi-2026-08-19-140455\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000900d00\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-d5pz9g\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-ttgbbn52wj5b\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f80b855b-6ddb-4d41-bcfd-69a629fa5f3a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f80b855b-6ddb-4d41-bcfd-69a629fa5f3a\",\n      \"name\": \"f80b855b-6ddb-4d41-bcfd-69a629fa5f3a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T23:46:03.25868454Z\",\n      \"endTime\": \"2026-08-19T23:46:13.75881135Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-17-010411 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:174]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-d5pz9g\" to 4.23.0-0.nightly-multi-2026-08-19-140455\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000900d00\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-d5pz9g\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-ttgbbn52wj5b\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f80b855b-6ddb-4d41-bcfd-69a629fa5f3a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f80b855b-6ddb-4d41-bcfd-69a629fa5f3a\",\n      \"name\": \"f80b855b-6ddb-4d41-bcfd-69a629fa5f3a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T23:46:03.25868454Z\",\n      \"endTime\": \"2026-08-19T23:46:13.75881135Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-17-010411 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090212357799153664","occurred_at":"2026-08-19T23:00:07Z","signature_id":"6e67a0588318a98d023efbba4cd0e38a13983e93b7d4dccafc848765ed882535","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"93eb9aed3c673a25dda3025e9f38f8d1c53f9c491622feaa90552b1afbad3297","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-20-141954 to 4.23.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:174]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-r7msf7\" to 4.23.0-0.nightly-multi-2026-08-20-085020\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000db4400\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-r7msf7\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-cxr6vwskbzqs\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cbae144c-9ad6-4399-8e1e-708ba5343570\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cbae144c-9ad6-4399-8e1e-708ba5343570\",\n      \"name\": \"cbae144c-9ad6-4399-8e1e-708ba5343570\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T23:45:35.456682564Z\",\n      \"endTime\": \"2026-08-20T23:45:36.715219027Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-20-141954 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:174]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-r7msf7\" to 4.23.0-0.nightly-multi-2026-08-20-085020\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000db4400\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-r7msf7\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-cxr6vwskbzqs\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cbae144c-9ad6-4399-8e1e-708ba5343570\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cbae144c-9ad6-4399-8e1e-708ba5343570\",\n      \"name\": \"cbae144c-9ad6-4399-8e1e-708ba5343570\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T23:45:35.456682564Z\",\n      \"endTime\": \"2026-08-20T23:45:36.715219027Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-20-141954 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090574634771550208","occurred_at":"2026-08-20T23:00:08Z","signature_id":"a5317b67ffb05362e28d05bcc9279b1afb1324ef6112b96fcdc34cfd4192cc2e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f9bfb1faf51b51475c30265bfb24f836c9386a47bad2413a3d8e330ed7c9045e","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-20-215447 to 4.23.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T23:00:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-7jwrlb\" to 4.23.0-0.nightly-multi-2026-08-21-150158\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00071fec0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-7jwrlb\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-ddtbb45g74fw\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/47800b8f-a8ff-4f7e-b7b2-0b0723bd00f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/47800b8f-a8ff-4f7e-b7b2-0b0723bd00f5\",\n      \"name\": \"47800b8f-a8ff-4f7e-b7b2-0b0723bd00f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T23:45:19.683338639Z\",\n      \"endTime\": \"2026-08-21T23:45:20.631024642Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-20-215447 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-7jwrlb\" to 4.23.0-0.nightly-multi-2026-08-21-150158\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00071fec0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-7jwrlb\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-ddtbb45g74fw\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/47800b8f-a8ff-4f7e-b7b2-0b0723bd00f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/47800b8f-a8ff-4f7e-b7b2-0b0723bd00f5\",\n      \"name\": \"47800b8f-a8ff-4f7e-b7b2-0b0723bd00f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T23:45:19.683338639Z\",\n      \"endTime\": \"2026-08-21T23:45:20.631024642Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-20-215447 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2090937171266506752","occurred_at":"2026-08-21T23:00:43Z","signature_id":"6ce3f21db4e03dbc02da614b27bd974fe76c85c6dcbc700f959708a7547d5026","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"9ae716254ca5ddb2b82f158d6a7b9ae7aabc6066c7efeaa8525d2090a51a812e","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-21-221014 to 4.23.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-22T23:00:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-z5pqzx\" to 4.23.0-0.nightly-multi-2026-08-22-063717\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012ac660\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-z5pqzx\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-227n7tlwrhnd\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ef2c5e8e-3ad7-4179-842e-7126874115be\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ef2c5e8e-3ad7-4179-842e-7126874115be\",\n      \"name\": \"ef2c5e8e-3ad7-4179-842e-7126874115be\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:47:18.723854025Z\",\n      \"endTime\": \"2026-08-22T23:47:19.754634857Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-21-221014 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-z5pqzx\" to 4.23.0-0.nightly-multi-2026-08-22-063717\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012ac660\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-z5pqzx\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-227n7tlwrhnd\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ef2c5e8e-3ad7-4179-842e-7126874115be\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ef2c5e8e-3ad7-4179-842e-7126874115be\",\n      \"name\": \"ef2c5e8e-3ad7-4179-842e-7126874115be\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T23:47:18.723854025Z\",\n      \"endTime\": \"2026-08-22T23:47:19.754634857Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-21-221014 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091299562504327168","occurred_at":"2026-08-22T23:00:44Z","signature_id":"6b3d20a396a74d44cdd8a7cc9268f6dc244330e48dd00a260ef2f088a769b44f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3b509588f678a0fd976eb453004a3e8a55c0be5c12f8340c463453a657ce6678","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-24-152026 to 5.1.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-q6hwb9\" to 5.1.0-0.nightly-multi-2026-08-24-111752\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a25da0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-q6hwb9\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-kgxwzfcjcc2b\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e23a315-fb36-4d2d-bc8a-8cee31204e59\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e23a315-fb36-4d2d-bc8a-8cee31204e59\",\n      \"name\": \"7e23a315-fb36-4d2d-bc8a-8cee31204e59\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:51:24.1449981Z\",\n      \"endTime\": \"2026-08-24T23:51:30.86207539Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-24-152026 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:217]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-q6hwb9\" to 5.1.0-0.nightly-multi-2026-08-24-111752\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a25da0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-q6hwb9\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-kgxwzfcjcc2b\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e23a315-fb36-4d2d-bc8a-8cee31204e59\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e23a315-fb36-4d2d-bc8a-8cee31204e59\",\n      \"name\": \"7e23a315-fb36-4d2d-bc8a-8cee31204e59\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:51:24.1449981Z\",\n      \"endTime\": \"2026-08-24T23:51:30.86207539Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 5.0.0-0.nightly-multi-2026-08-24-152026 to 5.1.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2092024382393159680","occurred_at":"2026-08-24T23:00:55Z","signature_id":"05526aa86d5e4dd680e08fb3cf223532281a30bb281575b5c8abfec129525d60","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c14df85165f7ab51105a1e05c6306a467e1e1edcac57869d6c32690164316413","failure_pattern":"VerifyVMOSDiskCustomerEncryption(nodePool=\u003cnodepool\u003e) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-z8ktk-l4d4d_OSDisk for VM des-encrypt-des-np-z8ktk-l4d4d has encryption type EncryptionAtRestWithPlatformKey, expected","search_query":"VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-z8ktk-l4d4d_OSDisk for VM des-encrypt-des-np-z8ktk-l4d4d has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-18T02:00:45Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:163]: cluster verification failed\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013905a0\u003e: \n    VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-z8ktk-l4d4d_OSDisk for VM des-encrypt-des-np-z8ktk-l4d4d has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    disk des-encrypt-des-np-z8ktk-t2cjq_OSDisk for VM des-encrypt-des-np-z8ktk-t2cjq has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:163]: cluster verification failed\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013905a0\u003e: \n    VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-z8ktk-l4d4d_OSDisk for VM des-encrypt-des-np-z8ktk-l4d4d has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    disk des-encrypt-des-np-z8ktk-t2cjq_OSDisk for VM des-encrypt-des-np-z8ktk-t2cjq has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089532925845442560","occurred_at":"2026-08-18T02:00:45Z","signature_id":"79876917e6824dcbd8ac712a77e0a6fb9ff34a76687e955fa8cb42c1b782a145","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"925c581adc4f4ee74421c479e1f09406ff5021fa3c71cc253a16495b01df4b7d","failure_pattern":"failed to update cluster \"\u003ccluster\u003e\" authorized CIDRs to include test runner IP","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-22T23:00:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:249]: failed to update cluster \"agg-cluster\" authorized CIDRs to include test runner IP\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:249]: failed to update cluster \"agg-cluster\" authorized CIDRs to include test runner IP\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2091299562504327168","occurred_at":"2026-08-22T23:00:44Z","signature_id":"da4948d61ab54e112557ad6fe48bbeb3135dc01594a49200da12544749a36fa1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":6,"also_in":["INT","STG"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":12,"prior_last_seen_at":"2026-08-19T02:00:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000d24198\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-xjt5s.apps.aro.cilium-cl.j5nd.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000d24198\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-xjt5s.apps.aro.cilium-cl.j5nd.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089895148526768128","occurred_at":"2026-08-19T02:00:06Z","signature_id":"6985a3af1e1b7d9f14fa3e6ebca829c6de310ae27552aab65799f5ab8b2a33dd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"7137d3f115bf4c264bb74c888bdc6c3954a56a37032931fb9aa8d47ff31a4953","failure_pattern":"x509: certificate signed by unknown authority","search_query":"x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.4482758620689653,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["INT","STG"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T10:34:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.002s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.002s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2090024653178277888","occurred_at":"2026-08-19T10:34:42Z","signature_id":"85944d4fa3cc8f9a02514637d6119a2fd69da50caa8d14fdae25e1e32d2d5f52","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"stg","summary":{"total_runs":26,"failed_runs":24,"raw_occurrences":261,"matched_occurrences":261,"runs_affected":24},"rows":[{"environment":"stg","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":55,"runs_affected":8,"failed_runs":8,"run_impact_percent":30.76923076923077,"anchor_occurrences":55,"after_last_push_count":55,"also_in":["INT","PROD"],"trend_counts":[0,0,0,2,10,17,18,8],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-07-27","2026-08-17"],"prior_runs_affected":9,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:196]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001990018\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-58vq7.apps.aro.cilium-cluster.xjbu.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:196]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001990018\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-58vq7.apps.aro.cilium-cluster.xjbu.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0023089a8\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-gd2m6.apps.aro.agg-cluster.sh9x.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:336]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0023089a8\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-gd2m6.apps.aro.agg-cluster.sh9x.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc002bc60a8\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-2xkbq.apps.aro.cilium-cl.yqg3.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:175]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc002bc60a8\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-2xkbq.apps.aro.cilium-cl.yqg3.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"128bdc5bd9fd4d676037c7b3876358423100506abaac1cb7c6a2aa6dca99f9d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"268af4cfefc9d9ce5477c6309e34d9e625934c5fec5612737f144e143459ca23","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"568397239d373e84cc1193077022f2929fdbc7536d41e0a023cda085a91923ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"64d7370897c1ddb0a4c082a98b5de0abd894ee5bd836383e651a1f53f91a033d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"6f400eea163c9e3d28c8b1f64accc1cf5963abf78c260413f5fe388a60f0006c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"aa4dc686c4f2ffb192f550321a5d618c671a84350ba22f26b4ee1aaac5686fb0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"c217516f086c4ee518f66b5fc0b63d7e19723573004fb051ee9ac9e2e83e0fb9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"de74f8de034a6e8da10a240f9e0d6f03ee3be2522666aaf0b6ad709ddf3efbc4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"0652b92117d22a5b2e9cf20aa5637d5c27db260e7c8ad186d851762e8f90ade7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"0c968f59cb717431d5c7d3b774ddd58e54f3c9e98d2283d5ed8c612c784ab28f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"22b5ae279ed4a4d36e707040d191eba08b365c5589624a6177f37632ecb6dd27","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"4e439067d1f887fd6586941fe0a591f509c53905e7b1e54cc808ee63599dd229","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"56a2d316ec317e48edc96d53b9c004ddce28b2b100d6c2fbc7d62acc3230859f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"81e44a56cd141302f440c9207b5cd0d22a87018ab48353bc76f47efbe9dc9da6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"8f6da9f6c6a1f82e13e70219a05bdb624dbe89ba1d87c0b4d0e1e9ad11d1b63d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"917a7ab84fc8483a4bd3b2ce2620cefe932a4a57e8830c6d4a4588716e952577","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"a9220eabfed81c9d6a05b1347966522bde35e0e1de1e1d5420f8bfe1ce00e1b3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"dff5d87b8dd7ae53a959d7abd4ee8e7071329d518e20c863334623937ac665a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091344896731385856","occurred_at":"2026-08-23T02:00:53Z","signature_id":"099c7f9ec6546f82588b173d8d978935aefdaf7c422c94012f4b5bcff2a2bd2f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091344896731385856","occurred_at":"2026-08-23T02:00:53Z","signature_id":"358e627c5a3717e340077769b0bd08ea4a344ecd196320db476ee3f70bc37bd4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091344896731385856","occurred_at":"2026-08-23T02:00:53Z","signature_id":"3b1f4a9168c319727c1d0cdb76f21a361aff2705a016f6e2bd6ce401fdc245a2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091344896731385856","occurred_at":"2026-08-23T02:00:53Z","signature_id":"4b0b2bdfb319f0241a615594f15b72f98d287b016595aa560833ca643e7c41b1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091344896731385856","occurred_at":"2026-08-23T02:00:53Z","signature_id":"4e525635a334f496f86e5447f7bfcc41a3027f81d279cac4a2c98a911e1a14e2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091344896731385856","occurred_at":"2026-08-23T02:00:53Z","signature_id":"b0305dfd29e4255374c19ac1807f6ec0adeadfa5faf330d938857c76cee28385","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091344896731385856","occurred_at":"2026-08-23T02:00:53Z","signature_id":"ca5231bfe441f0896ee14844376689aae71fdb84bd0ceb16d221be71b2d1f758","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091344896731385856","occurred_at":"2026-08-23T02:00:53Z","signature_id":"eef5d362c6ca5ec89cb57e6a06e53474bbe066166bdea8cc01dfe1eee398b32d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"11a725162d782d59307e54b01812f83276c8b768291c214985b0e7cf30d62295","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"239e965421538be1363dd2c7f07fdf5811ab39dbc83ef7b4d067112bb5d9f0fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"3cd272fe93bf03fdbe630ac01847905ab2758c75271e802033f6c152ac24510d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"473ae6affbd1614b127350a684f5c57cf48befc5b71ca9167deb53482d16e04e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"5aea9a96af15785f8a001a629201e85cfd8f31b12c83f95c9952b9431240bf9b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"5b0f787f537c20118ee76f9fc63fd06eb58ee0eee7e5ee4575316f4340472c37","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"7ffc802cb180f4209a0fa858155687cc6e2899d3764fe7d6bd1321ae4fd55e84","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"8c6b27faba09bc535bc427b19d9416db95aeff2cd1af3b110ae29985fbe189dc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"c5fb4bff03f86320f1edc3eea596547758a86968f42b91dffea826dc780ab8c0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"fdf84045828cfe2df5b8ebc4b5d8480c96306c6e7327c688db9c24bc34502f64","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"15cc422b2f3be8fc9e52950819b6911fd45c6c141421a6134c4d451278306bca","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"1b080e7bcd141068d87382a1bcd086ccda9b6dd57cad2be71077c408d2e7e438","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"7d8117c22146bf5a0774f739c19b87fd0cbd57d6a02884d31d96768d87ca983d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"907dec55a9074061a33272193d894502c57d11b65a9f5f49e62962d638dd0ba1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"9a9e1cb7ce25683e8a9ba03d52e031fa731f92999b78652a943b0a33011dc620","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"db6298b2d5a06ac0c0b6891a012777daa148bd9f9827d3298b5f01d7948507b1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"f0207e912b5808bacb31a7fe73aaf94e3c55660884e013054934fb98f7a335f8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"014faee8c65110da8bf653c059bda71ae8ef3480a383a2999fc04af379294b99","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"0e428625a39042ad70067f1e3a5ffb3ef2deec8808b6df01f291fa0854988be4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"115305b069b0aa8d64167995814d5cd6d32a1f792870e701a3affd4a8b96bdfe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"587e72c577de17291ef8b6b3ac4f6b12a0663bf2c56e0f53f30373aec8471091","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"6073bf034d1aabb50f60a9559998d196f90b404e7a6f2c7c9d69ac4bbe1020b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"697a4c129533a315c9016b84da122b5bc17c7516225d6ed78343e011c45dd4ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"69ff046ff55d8bfc7023c374da3602741135d47054bcf695f149cb3279b1d92b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"a1166de0889c8bcb931d1a2474d9ac58b2df20920fe3d4ae0d4b27104be80122","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"a610245db2dc06bccba732a90d5f0ee6d5e2651fd4bb3d357ae0d19e0833cec8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"bf0b84a697d66c570dffb08e752d5c50734534e54f512951dc11b3757ac8cd8e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"4649980ccb572a8df4fef4e44831f1de4eae2d176d8ecffa92fb99f1ae87aa7d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","signature_id":"bf90b8b00390291912ae577c4eaa041b4552832f8c625cf2cd2cff628c244b5a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":47,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":47,"after_last_push_count":137,"also_in":["PROD"],"trend_counts":[12,47,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-18T02:00:45Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b26d20\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-rxs7l9wkbj9x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/aceae8b1-ad97-4fd9-85eb-b3b27c9aebb7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/aceae8b1-ad97-4fd9-85eb-b3b27c9aebb7\",\n      \"name\": \"aceae8b1-ad97-4fd9-85eb-b3b27c9aebb7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T02:02:45.64795053Z\",\n      \"endTime\": \"2026-08-18T02:21:50.493813123Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b26d20\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-rxs7l9wkbj9x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/aceae8b1-ad97-4fd9-85eb-b3b27c9aebb7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/aceae8b1-ad97-4fd9-85eb-b3b27c9aebb7\",\n      \"name\": \"aceae8b1-ad97-4fd9-85eb-b3b27c9aebb7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T02:02:45.64795053Z\",\n      \"endTime\": \"2026-08-18T02:21:50.493813123Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa8840\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-zdptt7lrhq2v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8ed61913-8f7b-4249-8b7b-d53652b4a639\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8ed61913-8f7b-4249-8b7b-d53652b4a639\",\n      \"name\": \"8ed61913-8f7b-4249-8b7b-d53652b4a639\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T03:33:58.580313157Z\",\n      \"endTime\": \"2026-08-18T03:53:04.009688419Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fa8840\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-zdptt7lrhq2v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8ed61913-8f7b-4249-8b7b-d53652b4a639\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8ed61913-8f7b-4249-8b7b-d53652b4a639\",\n      \"name\": \"8ed61913-8f7b-4249-8b7b-d53652b4a639\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T03:33:58.580313157Z\",\n      \"endTime\": \"2026-08-18T03:53:04.009688419Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b151a0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-5vv87vj8ff2q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/09fc3a7a-6044-4ad3-857e-0da0e928149d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/09fc3a7a-6044-4ad3-857e-0da0e928149d\",\n      \"name\": \"09fc3a7a-6044-4ad3-857e-0da0e928149d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T02:49:17.430704094Z\",\n      \"endTime\": \"2026-08-18T03:08:22.123216944Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b151a0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-5vv87vj8ff2q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/09fc3a7a-6044-4ad3-857e-0da0e928149d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/09fc3a7a-6044-4ad3-857e-0da0e928149d\",\n      \"name\": \"09fc3a7a-6044-4ad3-857e-0da0e928149d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T02:49:17.430704094Z\",\n      \"endTime\": \"2026-08-18T03:08:22.123216944Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"030afe4f177e04334c985bcfa72e16b8be261223fba47b3c5763991c7e524b35","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"040f6603d8b438d217fbab67465748837dce73051f11ae19c099294986a82d63","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"0ef26d0c51d96d0177563c34ab8c7a18349b584f0533d11492bda7f527fc77c0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"17ed13a24ab02c5de665afad5bee1b614714b7cf93a209fc462fa05344b0bee4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"1c0da96ae4f4dc2661ff24cee7bbeb9699f5a3519fd0d8f002fecb084979d146","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"211ccfe0ba4e436ba76c27d568452f5b4e8838df0a37c676b4da68875224d176","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"21db64526cd3217b2a78996248ff29f0b453d2891008a8ed119cb7cd778361ef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"2ca5bbde35a04e18a54077be69052f1e54f6b92f3b4c84b0e1e6ee66f33a4fb5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"2d7a43846ce13d0ef0f59b9ddbfaaa7746145459b4c9ad36d9da514cd624697c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"2dba77a5099febca0d2e632f7342c72e8ff35d04b4b63c2c9ccb180c8f05c982","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"3698dec1690914ee04d60fc6057b3363816b6223a6ad4f48470df16612dbc007","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"3b9339d87915b7d23e6e47d682efce6ea1179b3ee9c9c5ce7c58f07a35d7a953","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"3d338f18b9791588b06b34a8a45a8addace76cf65a1ef2e2683384af83defff4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"3ffeaa534d07b8ac423544eb9f309a8153bd272f3374686f2d85ac99dca6fd9a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"4921271e4084b1c96811719733d2179d78e8188b990b071e2550739f701daff5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"4efc237be38db0998ab7200738e8594982710f04340016e069b7010bcf130cc5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"524a1a06830044bd9a54ead309a29897b902eea86204671d3a07e12b6d246f16","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"526d8b80b44f64a1e869a97e2dae959cbe880639f92bc9cc16df14e0da53a9ba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"591510aaf242ea6c4381fef95a4a01be0e024570e5a9eab771a7eb31232764f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"5b644019d0c730e1955b30a11efc46efd83e52d040c79898b2938460997cabb3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"5dcfe1ac8140a4de9e7320d281d7d75ddee940144d63955b4d089dc50bf5f95f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"5ddee523c1876f95b665208a851b55a9958ce3fafd15ef8e07fafc3fbc030ab8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"7118975628f6c4101d2fc3e6ee041a7c9ce6694b172c74fb3c937c59986b75c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"80a433a76b4973b57477a38b0603b93e5ff0257d60b1670501bb6a625720d09e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"8fa7888e460857cc2280931a6c3194b76b79e78422a93e71a6ebc8533bb7cdde","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"91eb38aaefbdfca743b679b959430fa2a4ad8124e50bebe3e76242b19e3302f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"994ccdc96407ba616a152c182cc84ab9ac9db7587d950ffaef9453c5e081be8b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"999385dde7e66d753738268538577e8ec929327d69680d733d9cb4aadd490e2b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"a48ac948e8f62adc203810b7d8cac91933840123e0460b2aed992461e22280d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"b11c514a2ea2cf8ab315946847a8ee6c82b9b1a7a117061de8e283ee667333c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"b3395406faa347361e3d7589a7dd4c40a7ad83a82f27fe779877f9636220a1f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"b3ba5a21f5c7f9628b5e5e5d56923eca937e8a7031ababe3f3728c3f4431b058","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"b696cfcf46e4ed451540b3371b67da873ec9cb51fee295b7aec92e8b16453f8e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"b6b714e5fa37be1b94c97aaa42aa90bd08ffedefed3d499738ea3bcfce39bda9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"b8adea9d7d194afaf613750299d712b7eb9510d20d2071dbe62354400a882669","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"bbb40ac860a5524013de133fe007897223067a2f050e5b90c997770dea341f60","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"bca77ee1607a00938ae5ce1f661e70d3ecb5fca7aea21f5da331eec910379dd9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"c275f880c385321671b31ddc6a798486ab95e3f2038a1fe58331a3d86cfdf19e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"ced8793055601f23ff5fe306d0b2b63d8dd4366e8a3ea6fba0641bfc4ec37924","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"cf3a1ad571b67c5fc4e3fefd8b622a71909a2400fb1c26ae47eeefdca9e71154","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"cf97614afa7d5fc7c111167c9cbb62a5202a2f493906837d3f91f3ba2ac42804","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"d4a48fc4a05fe7280f01a607af223f52f2d5bb0df076e4c641294ecce678bed7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"db86d9275849ee710324ab2feed28953853d50a8df3a2ae4d0b48cd27737eb7f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"df5eed79a8067b92db69ac5eb2a91ae634782906d1265426253e8a8b791a4618","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"e0326f3347faeae8674d9fbab1fe60a2816c9695baafb5b722bdda3f729db2ef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"f3dba0b5afe5199331ae44d4d6742df5d4a3e18e9c91a5db239fd862390ae3aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"fd4d758f60623a497bbe24e1633ead98211617d8ec7e516e627a30586c6c9df3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":19,"runs_affected":5,"failed_runs":5,"run_impact_percent":19.23076923076923,"anchor_occurrences":19,"after_last_push_count":19,"also_in":["DEV","INT","PROD"],"trend_counts":[0,0,3,0,4,4,4,4],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0012c61b0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22.0-0.nightly-multi-2026-08-23-080550, targetMinor=4.23.0-0.nightly-multi-2026-08-24-110923) failed: clusterversion status.history has no version in target minor \"4.23.0-0.nightly-multi-2026-08-24-110923\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0012c61b0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22.0-0.nightly-multi-2026-08-23-080550, targetMinor=4.23.0-0.nightly-multi-2026-08-24-110923) failed: clusterversion status.history has no version in target minor \"4.23.0-0.nightly-multi-2026-08-24-110923\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000013c68\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-08-24-135049, targetMinor=4.21.0-0.nightly-multi-2026-08-24-143120) failed: clusterversion status.history has no version in target minor \"4.21.0-0.nightly-multi-2026-08-24-143120\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000013c68\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-08-24-135049, targetMinor=4.21.0-0.nightly-multi-2026-08-24-143120) failed: clusterversion status.history has no version in target minor \"4.21.0-0.nightly-multi-2026-08-24-143120\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001423c38\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-24-143120, targetMinor=4.22.0-0.nightly-multi-2026-08-23-080550) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-23-080550\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:226]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001423c38\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-24-143120, targetMinor=4.22.0-0.nightly-multi-2026-08-23-080550) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-08-23-080550\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"1f961d63e5efbc9f674297ab6ff360e18191616cad68297a32bfc0352b1efd45","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"5d51d6400541f1ef2ea735bc34f88c2cca7f1b91d8f8680efed7dfe0ab6e075d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"8c228d79217e5703c43c091d1bbcb6c74c0d2e48dd70a8ed3bbc113b8600d236","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"9a6af90baac0bf0367bb530e542307559248b48af09dc4fa5550242ec09f64e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"4130f29b010aeb34f9c8f43a05c2e44d7785c514f2add915cf23a436d9b27ff7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"7a1d942e87a8503eedbb424b862f5ab7d117d1ff924fa3f4d4d4d997e3166d93","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"9cf68146343a69d8617ea02654d3caa9f50c984ec8e49043fd90de1d043d4cef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"e8776a142c237294cf872e39df27cf70f10aa1c9176e12b51594515a2d850deb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"444e0b46276d6b06038a0349a8a8b1b9c6bd0d97e65d53c08b2f7737e27ce63b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"6183ddd6caf2d7afde8bcd7be49b7fe37da55024a657dec1d775306e9477f62a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"87802490969f4a6d1eb8eb8df9ca6f5f32bcb5b82112d8d6b1e304b1d96e3e03","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"b7e9a22c6cf3ac389f2a77fd9b2625cddcf6fd44e337c61d636e8df2c2743c61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"62bbc0ad65ca651617994b83d05a4a4fe629d762c5e177de75e71c5449cc78fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"86e1eb29ec597937a663760329209bb50694b5af2a87b7c19ee3c634d1801423","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"b880458c73aad38c09b6646920150ac274a35d4170590ae9673504a08162d18b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"e6a8ea22f064439192de7c94bd0b019d27fcf10b53a96a3dbbeaabdae0a1680a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090212361167179776","occurred_at":"2026-08-19T23:00:07Z","signature_id":"1d432859b9b61eadeae3f59e9810dbfd5dbaf0c705a7bb7f6f36b66b56115528","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090212361167179776","occurred_at":"2026-08-19T23:00:07Z","signature_id":"2695f4fff945203d2d453601faacf71292783cffa1d1c099df7d765bed100038","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090212361167179776","occurred_at":"2026-08-19T23:00:07Z","signature_id":"b5edad6c45e84113c6ceffd16aae87af35a9ee427b9bd298e22a3b075a317a70","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-5-0-nrmbvd, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-nrmbvd in resource group rg-candidate-5-0-dgkn5p9l9flc, error: failed waiting for cluster=\"cluster-candidate-5-0-nrmbvd\" in resourcegroup=\"rg-candidate-5-0-dgkn5p9l9flc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-nrmbvd in resource group rg-candidate-5-0-dgkn5p9l9flc, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":11,"runs_affected":9,"failed_runs":9,"run_impact_percent":34.61538461538461,"anchor_occurrences":11,"after_last_push_count":11,"also_in":["DEV","PROD"],"trend_counts":[0,3,2,2,1,1,1,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":26,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":7}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002ac4480\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-gp6bzmxwxfps\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002ac4480\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-gp6bzmxwxfps\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-gp6bzmxwxfps, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001a9cb70\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-4mnnfzb2nhtb, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-4mnnfzb2nhtb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-4mnnfzb2nhtb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001a9cb70\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-4mnnfzb2nhtb, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-4mnnfzb2nhtb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-4mnnfzb2nhtb, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002b9e330\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-fg554p8tjmmf, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-fg554p8tjmmf\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-fg554p8tjmmf, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002b9e330\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-fg554p8tjmmf, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-fg554p8tjmmf\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-fg554p8tjmmf, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"1ecfdcaa8fd2583c7f65498931069ae0e42fa1b933a7ed7714a974ea82a7b373","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"5554100c5cb6fda2b51ccf73571621c9937fb5b9897851e954053e5c8912e0a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"4f5c940afced23891d414938cb4cb38d59030d0411748cce89554393d71b5e2c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"9d8ff92f48ab455bb4637f037f6fb65f0f9eab5e788e3363f3d8f0a842a4a77d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"5b48ea80fc398a034dba3d46b19e592386ab5e5708b0a0e23267c0282b5a55be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090257540762832896","occurred_at":"2026-08-20T02:00:07Z","signature_id":"2c025bacef2a95219db5cc2c1d0f8e25156096d570a86e526e43447bf8f9a29b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090212361167179776","occurred_at":"2026-08-19T23:00:07Z","signature_id":"6f01fa48d2614fdfbfe9c406cf323639e0a1db4176d42dd1853204a6a7a20910","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090212361167179776","occurred_at":"2026-08-19T23:00:07Z","signature_id":"ba0f03a868ed9f5c8c401dbf4f120f2223529a931c2927647bf01ebba2dbf3c4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089849852383989760","occurred_at":"2026-08-18T23:00:06Z","signature_id":"7f6a383316c448cc55b3216acf6cd57f15238910a191957b115b3e19ac925813","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089849852383989760","occurred_at":"2026-08-18T23:00:06Z","signature_id":"d300dfa60eae7d130738a31cc816ef7cabf50ea87f8f008cf9c7921b07a9fd85","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"3d17a875fdfdb8f9b420ebc4b2a321d93c525a23e3d7f70a3d6bc16e1d0a8f11","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":10,"runs_affected":8,"failed_runs":8,"run_impact_percent":30.76923076923077,"anchor_occurrences":10,"after_last_push_count":10,"also_in":["DEV","PROD"],"trend_counts":[0,0,4,4,2,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":8,"prior_last_seen_at":"2026-08-21T23:00:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011964b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-h8fb74\" in resourcegroup=\"rg-np-version-upgrade-h8fb74-s5mqnw56gx4w\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/701376c3-ab04-47a8-a1ef-8375724f8f86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/701376c3-ab04-47a8-a1ef-8375724f8f86\",\n      \"name\": \"701376c3-ab04-47a8-a1ef-8375724f8f86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T00:26:59.095965518Z\",\n      \"endTime\": \"2026-08-22T00:51:01.463363445Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sb99fkh6uh3oo5nr4qni7jhed5o0tgt still exists (deletion dispatched at 2026-08-22T00:26:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011964b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-h8fb74\" in resourcegroup=\"rg-np-version-upgrade-h8fb74-s5mqnw56gx4w\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/701376c3-ab04-47a8-a1ef-8375724f8f86\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/701376c3-ab04-47a8-a1ef-8375724f8f86\",\n      \"name\": \"701376c3-ab04-47a8-a1ef-8375724f8f86\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T00:26:59.095965518Z\",\n      \"endTime\": \"2026-08-22T00:51:01.463363445Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sb99fkh6uh3oo5nr4qni7jhed5o0tgt still exists (deletion dispatched at 2026-08-22T00:26:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0013c8138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-2p9lgg975wsf\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5710f93-c135-4ecd-84ba-bd766a4f3157\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5710f93-c135-4ecd-84ba-bd766a4f3157\",\n      \"name\": \"d5710f93-c135-4ecd-84ba-bd766a4f3157\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:51:19.851300811Z\",\n      \"endTime\": \"2026-08-21T03:15:24.947768689Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2samnf949m9mks8utf8hvjl3i9ocmvd0 still exists (deletion dispatched at 2026-08-21T02:51:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0013c8138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-2p9lgg975wsf\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5710f93-c135-4ecd-84ba-bd766a4f3157\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5710f93-c135-4ecd-84ba-bd766a4f3157\",\n      \"name\": \"d5710f93-c135-4ecd-84ba-bd766a4f3157\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:51:19.851300811Z\",\n      \"endTime\": \"2026-08-21T03:15:24.947768689Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2samnf949m9mks8utf8hvjl3i9ocmvd0 still exists (deletion dispatched at 2026-08-21T02:51:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011f24f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listwvkfxb\" in resourcegroup=\"ea-list-q8zktqx9k8tx\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e8f24bc-6934-4503-b388-7fea73015050\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e8f24bc-6934-4503-b388-7fea73015050\",\n      \"name\": \"7e8f24bc-6934-4503-b388-7fea73015050\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T14:04:49.548371908Z\",\n      \"endTime\": \"2026-08-20T14:28:52.367967433Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sabeurund3r70lf12nj56m098p5j7j9 still exists (deletion dispatched at 2026-08-20T14:04:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011f24f8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listwvkfxb\" in resourcegroup=\"ea-list-q8zktqx9k8tx\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e8f24bc-6934-4503-b388-7fea73015050\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e8f24bc-6934-4503-b388-7fea73015050\",\n      \"name\": \"7e8f24bc-6934-4503-b388-7fea73015050\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T14:04:49.548371908Z\",\n      \"endTime\": \"2026-08-20T14:28:52.367967433Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sabeurund3r70lf12nj56m098p5j7j9 still exists (deletion dispatched at 2026-08-20T14:04:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"ca75b2343ff23a11e0fce1a31c9ef24459c4c103d790af5c0d8320fc327b3727","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"afbd9ea61b7f5f2cf2d4e5eb07c2fecfd15d1778af4a847a49ff12ce110071ab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","signature_id":"c98eeb6d2064c80a5c88dac50a9f84605cf8ee1eb70a70345435f7284fcc94c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"0facf4aca2cbbc09d1d4d2b5f2a2c443dc9ac7bce839a50cfc50190997e06fd4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"a1399ba06ca55ef9ac4602a99872b2b1d13798c7ec52bf26477719dd69c14e17","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090257540762832896","occurred_at":"2026-08-20T02:00:07Z","signature_id":"d19ec4d4f31d760d02b50b389f562a65f23856356a3f633669a10c48565d949d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090159303129305088","occurred_at":"2026-08-19T19:29:45Z","signature_id":"e3d4b835e521fff377b185d308d1607a3e78fb8712e5a6bf8fcb1ba092f6e484","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090159303129305088","occurred_at":"2026-08-19T19:29:45Z","signature_id":"e4f8b47f31271f0cf76b5df5def7ef54ddf1770bbdfd048bee2fd57247eec8de","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090064577516539904","occurred_at":"2026-08-19T13:13:21Z","signature_id":"c952a9e95cca1027c298ae780c02d302a431ed86a3d72fe39ce0c575ad12c319","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089951420068401152","occurred_at":"2026-08-19T05:43:42Z","signature_id":"0f4eaab480b1c780fe060d154f73a8d802037c158a76021e9a8d565c30c9bbe5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"819462e5c945d35c3bda97a6aae4ae2bf1db2fba1cd848aee697a026ed6e2265","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":9,"runs_affected":3,"failed_runs":3,"run_impact_percent":11.538461538461538,"anchor_occurrences":9,"after_last_push_count":9,"also_in":["DEV"],"trend_counts":[0,0,0,3,6,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:140]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-xbp8kt\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d57060\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-xbp8kt: failed waiting for cluster=\"cp-ystream-upgrade-4-21-xbp8kt\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-mvbldsj9x64s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0b50fbb-5e08-405c-bc27-320fd18ae286\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0b50fbb-5e08-405c-bc27-320fd18ae286\",\n      \"name\": \"d0b50fbb-5e08-405c-bc27-320fd18ae286\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:33:00.457364861Z\",\n      \"endTime\": \"2026-08-21T02:52:03.680608781Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, control-plane-pki-operator, kube-scheduler, azure-cloud-controller-manager, cluster-node-tuning-operator, ingress-operator, ignition-server, olm-operator, machine-approver, olm-collect-profiles, openshift-apiserver, cluster-policy-controller, csi-snapshot-controller-operator, catalog-operator, packageserver, community-operators-catalog, cluster-autoscaler, cluster-version-operator, certified-operators-catalog, openshift-route-controller-manager, cluster-storage-operator, cluster-network-operator, dns-operator, konnectivity-agent, redhat-operators-catalog, kube-controller-manager, openshift-controller-manager, hosted-cluster-config-operator, ignition-server-proxy, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:140]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-xbp8kt\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d57060\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-xbp8kt: failed waiting for cluster=\"cp-ystream-upgrade-4-21-xbp8kt\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-mvbldsj9x64s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0b50fbb-5e08-405c-bc27-320fd18ae286\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0b50fbb-5e08-405c-bc27-320fd18ae286\",\n      \"name\": \"d0b50fbb-5e08-405c-bc27-320fd18ae286\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:33:00.457364861Z\",\n      \"endTime\": \"2026-08-21T02:52:03.680608781Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, control-plane-pki-operator, kube-scheduler, azure-cloud-controller-manager, cluster-node-tuning-operator, ingress-operator, ignition-server, olm-operator, machine-approver, olm-collect-profiles, openshift-apiserver, cluster-policy-controller, csi-snapshot-controller-operator, catalog-operator, packageserver, community-operators-catalog, cluster-autoscaler, cluster-version-operator, certified-operators-catalog, openshift-route-controller-manager, cluster-storage-operator, cluster-network-operator, dns-operator, konnectivity-agent, redhat-operators-catalog, kube-controller-manager, openshift-controller-manager, hosted-cluster-config-operator, ignition-server-proxy, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f0f080\u003e: \n    failed to create HCP cluster np-downgrade-kjw6pt: failed waiting for cluster=\"np-downgrade-kjw6pt\" in resourcegroup=\"rg-np-downgrade-kjw6pt-lf69f9frnjjp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98f96261-7c0c-4604-82d2-ccf83d040db2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98f96261-7c0c-4604-82d2-ccf83d040db2\",\n      \"name\": \"98f96261-7c0c-4604-82d2-ccf83d040db2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:48:21.62730862Z\",\n      \"endTime\": \"2026-08-21T03:07:24.73168706Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-storage-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, openshift-route-controller-manager, cluster-policy-controller, ingress-operator, cluster-image-registry-operator, certified-operators-catalog, konnectivity-agent, catalog-operator, redhat-operators-catalog, control-plane-pki-operator, hosted-cluster-config-operator, packageserver, community-operators-catalog, kube-scheduler, azure-cloud-controller-manager, openshift-apiserver, cluster-version-operator, dns-operator, olm-operator, olm-collect-profiles, kube-controller-manager, cluster-network-operator, redhat-marketplace-catalog, cluster-autoscaler, machine-approver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:581]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f0f080\u003e: \n    failed to create HCP cluster np-downgrade-kjw6pt: failed waiting for cluster=\"np-downgrade-kjw6pt\" in resourcegroup=\"rg-np-downgrade-kjw6pt-lf69f9frnjjp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98f96261-7c0c-4604-82d2-ccf83d040db2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98f96261-7c0c-4604-82d2-ccf83d040db2\",\n      \"name\": \"98f96261-7c0c-4604-82d2-ccf83d040db2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:48:21.62730862Z\",\n      \"endTime\": \"2026-08-21T03:07:24.73168706Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-storage-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, openshift-route-controller-manager, cluster-policy-controller, ingress-operator, cluster-image-registry-operator, certified-operators-catalog, konnectivity-agent, catalog-operator, redhat-operators-catalog, control-plane-pki-operator, hosted-cluster-config-operator, packageserver, community-operators-catalog, kube-scheduler, azure-cloud-controller-manager, openshift-apiserver, cluster-version-operator, dns-operator, olm-operator, olm-collect-profiles, kube-controller-manager, cluster-network-operator, redhat-marketplace-catalog, cluster-autoscaler, machine-approver; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fbc300\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-dnkv5vg5t2ts\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/abfca9a0-da31-4fda-b740-f92e856a7dca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/abfca9a0-da31-4fda-b740-f92e856a7dca\",\n      \"name\": \"abfca9a0-da31-4fda-b740-f92e856a7dca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:36.161779463Z\",\n      \"endTime\": \"2026-08-21T02:22:42.321981114Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, cluster-node-tuning-operator, csi-snapshot-controller-operator, ignition-server-proxy, cluster-policy-controller, azure-cloud-controller-manager, cluster-storage-operator, konnectivity-agent, olm-collect-profiles, redhat-marketplace-catalog, certified-operators-catalog, packageserver, community-operators-catalog, machine-approver, control-plane-pki-operator, ignition-server, openshift-controller-manager, dns-operator, cluster-autoscaler, kube-controller-manager, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-network-operator, cluster-version-operator, ingress-operator, cluster-image-registry-operator, catalog-operator, olm-operator, openshift-apiserver, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fbc300\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-dnkv5vg5t2ts\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/abfca9a0-da31-4fda-b740-f92e856a7dca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/abfca9a0-da31-4fda-b740-f92e856a7dca\",\n      \"name\": \"abfca9a0-da31-4fda-b740-f92e856a7dca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:36.161779463Z\",\n      \"endTime\": \"2026-08-21T02:22:42.321981114Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-scheduler, cluster-node-tuning-operator, csi-snapshot-controller-operator, ignition-server-proxy, cluster-policy-controller, azure-cloud-controller-manager, cluster-storage-operator, konnectivity-agent, olm-collect-profiles, redhat-marketplace-catalog, certified-operators-catalog, packageserver, community-operators-catalog, machine-approver, control-plane-pki-operator, ignition-server, openshift-controller-manager, dns-operator, cluster-autoscaler, kube-controller-manager, openshift-route-controller-manager, hosted-cluster-config-operator, cluster-network-operator, cluster-version-operator, ingress-operator, cluster-image-registry-operator, catalog-operator, olm-operator, openshift-apiserver, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"0ed7bae60681c60f68c493d90cb1fd137349d28e37ec331f94e67f0f0671c10a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"238e7b24b648c5d6d6127862df062515aa947e0f889da3e18d7d7c8417d0316a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"32a0bd18972a8f8e83478af297e9368419e1b7e7597a308653070e30fe4fe6c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"37d9e7ef301047c2b2fd4899c663861930478c9a873b27090cbd860df00a8a91","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"950772107db22c8db323365cbb8dc7ad54495494533bdb154fc48dbef42f844b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"eaae9ffa124a963d7a6edae9a93b0de32ff8eeb2a17d21c5b6b88689e4aad9b4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","signature_id":"fd1a222e88fac51899884139c985b18a14bda354bbdc6d03b6bd0f1ec3b9916a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","signature_id":"fd7348293825c9d41446d338b7d31ffd23035dfeb87dc5ea60ab54b00dd1a82a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"f86b3413a4fa89f0fe3aee0a97c3b3aae586cbd24a62a162177322eec5c926c9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"acf4c4a2d46e5ea4b872812ba9f4a18d449fa2e31dbb376d0d126db4afe5c289","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":8,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":8,"after_last_push_count":8,"trend_counts":[0,0,0,0,8,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cc9300\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-6cppfxw8g4qh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e5d7c5e-2952-42c3-bb62-e707bfb32ce1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e5d7c5e-2952-42c3-bb62-e707bfb32ce1\",\n      \"name\": \"1e5d7c5e-2952-42c3-bb62-e707bfb32ce1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:28.697047474Z\",\n      \"endTime\": \"2026-08-21T02:22:32.526910358Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cc9300\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-6cppfxw8g4qh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e5d7c5e-2952-42c3-bb62-e707bfb32ce1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e5d7c5e-2952-42c3-bb62-e707bfb32ce1\",\n      \"name\": \"1e5d7c5e-2952-42c3-bb62-e707bfb32ce1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:28.697047474Z\",\n      \"endTime\": \"2026-08-21T02:22:32.526910358Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f84520\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-fq28q2k4mqnr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7022aa18-ac7c-4bd8-b67a-9fb51bb8cb51\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7022aa18-ac7c-4bd8-b67a-9fb51bb8cb51\",\n      \"name\": \"7022aa18-ac7c-4bd8-b67a-9fb51bb8cb51\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:19.705086647Z\",\n      \"endTime\": \"2026-08-21T02:22:22.572455462Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f84520\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-fq28q2k4mqnr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7022aa18-ac7c-4bd8-b67a-9fb51bb8cb51\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7022aa18-ac7c-4bd8-b67a-9fb51bb8cb51\",\n      \"name\": \"7022aa18-ac7c-4bd8-b67a-9fb51bb8cb51\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:19.705086647Z\",\n      \"endTime\": \"2026-08-21T02:22:22.572455462Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-wlltck9mnpdh/cluster-candidate-4-21-wqbh9c should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e0c460\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-wqbh9c: failed waiting for cluster=\"cluster-candidate-4-21-wqbh9c\" in resourcegroup=\"rg-candidate-4-21-wlltck9mnpdh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/61fe2608-61d1-400b-90c7-806bb5489f2b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/61fe2608-61d1-400b-90c7-806bb5489f2b\",\n      \"name\": \"61fe2608-61d1-400b-90c7-806bb5489f2b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:28.583859524Z\",\n      \"endTime\": \"2026-08-21T02:22:32.536144456Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-wlltck9mnpdh/cluster-candidate-4-21-wqbh9c should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e0c460\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-wqbh9c: failed waiting for cluster=\"cluster-candidate-4-21-wqbh9c\" in resourcegroup=\"rg-candidate-4-21-wlltck9mnpdh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/61fe2608-61d1-400b-90c7-806bb5489f2b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/61fe2608-61d1-400b-90c7-806bb5489f2b\",\n      \"name\": \"61fe2608-61d1-400b-90c7-806bb5489f2b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:28.583859524Z\",\n      \"endTime\": \"2026-08-21T02:22:32.536144456Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"0c797b790f34212a0962c3e54028dc04c84df9396c80443411635e702cab7abf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"43f28e097ff6599e427ad726f1fa1c75940fb0b825513844f3957b76cf6b817b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"565fce7fcb4445a79f30034d53708e522a56029627f0a6e682c9320663ffa91f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"6a04db7e50f70ea9dac9932acea60a13f6c9b7d4f8617adb14a39552f096a365","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"81fc71e1b2add474a2f9142629a0d6332c8b66a31ce3ed781eff6cdd129c1fb1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"82fc9dfcd4a1d1dc958933049664f71eaf4c60ebd9aac5ce72d39c5595e18a42","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"b84e75c4953bfe9005b85e725e144df6a003f1fa4344d9a74c269a1620ea8307","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"dc0cc1f46b4c3a94844aceabaa9ad9108ada785e52f771e817b2071398fc7bb1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3b73b158de2cfa00931d006d7dea813139c9ea99ca292b8ded3d10d4beb4674f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":8,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":8,"after_last_push_count":9,"also_in":["DEV","INT","PROD"],"trend_counts":[0,0,0,0,8,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001a80bd0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0a6ef0ec-688d-48ac-96fb-22ede87538f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0a6ef0ec-688d-48ac-96fb-22ede87538f5\",\n      \"name\": \"0a6ef0ec-688d-48ac-96fb-22ede87538f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:04:28.992322331Z\",\n      \"endTime\": \"2026-08-21T02:23:32.540875768Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001a80bd0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0a6ef0ec-688d-48ac-96fb-22ede87538f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0a6ef0ec-688d-48ac-96fb-22ede87538f5\",\n      \"name\": \"0a6ef0ec-688d-48ac-96fb-22ede87538f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:04:28.992322331Z\",\n      \"endTime\": \"2026-08-21T02:23:32.540875768Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cc1940\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-p8kxn92zjqhp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5185baff-cb0e-4088-baa6-c6571cadd51f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5185baff-cb0e-4088-baa6-c6571cadd51f\",\n      \"name\": \"5185baff-cb0e-4088-baa6-c6571cadd51f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:36.75245159Z\",\n      \"endTime\": \"2026-08-21T02:22:42.448207861Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cc1940\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-p8kxn92zjqhp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5185baff-cb0e-4088-baa6-c6571cadd51f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5185baff-cb0e-4088-baa6-c6571cadd51f\",\n      \"name\": \"5185baff-cb0e-4088-baa6-c6571cadd51f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:36.75245159Z\",\n      \"endTime\": \"2026-08-21T02:22:42.448207861Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000edcea0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-k8fl8dx2bw22\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/42d6143a-e97b-4483-afd9-f74bce1b7086\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/42d6143a-e97b-4483-afd9-f74bce1b7086\",\n      \"name\": \"42d6143a-e97b-4483-afd9-f74bce1b7086\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:44:46.647452675Z\",\n      \"endTime\": \"2026-08-21T03:03:54.737093461Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000edcea0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-k8fl8dx2bw22\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/42d6143a-e97b-4483-afd9-f74bce1b7086\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/42d6143a-e97b-4483-afd9-f74bce1b7086\",\n      \"name\": \"42d6143a-e97b-4483-afd9-f74bce1b7086\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:44:46.647452675Z\",\n      \"endTime\": \"2026-08-21T03:03:54.737093461Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"001ed7fa69d2a092f2b072655683cfb30462642d1bd846ab3b9b253dbcf3dab5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"0027226a133920ef1c26470ce81c33d48a4b562da304b31b0fe5ea881d2172b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"684f37cf48d101c235cdeb5cebaf623acfb01e63bf09a5dc4ed857bd9fa46e14","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"735a2a35f6b1c5e1b700cd4fe89f07201e4b403b4fd45e0a13a0acfcf9fe12f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"942360ff8169c2a19cebe856cddf17d577b93e9f7408dcd1cd1902c9daf1c0df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"ae9917932f90b2d8b738c6ed2ec73fd085b17fa53168e8719072bde1b10a9a0f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"b0b84b753e0f678ce469ee4fc8400cad30e638a19fedbd6f7746aad6ef378b5a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"f2ec5a1a25d21e5dec2ed93eab92a26a91c080ad53e104d71fabc59db7c4d615","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3bfb93ffa2915664d7de20fc0b753c1a5307e53691a4b8098db3a7e206ca4906","failure_pattern":"x509: certificate signed by unknown authority","search_query":"x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":26.923076923076923,"anchor_occurrences":7,"after_last_push_count":7,"also_in":["INT","PROD"],"trend_counts":[0,0,0,1,1,2,2,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":7,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.001s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.001s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.002s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.002s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.000s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.000s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"55d4b1840d69212b2703b61209abbc219f27e5b689e1fef7a10a1d77779d4060","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"85944d4fa3cc8f9a02514637d6119a2fd69da50caa8d14fdae25e1e32d2d5f52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091344896731385856","occurred_at":"2026-08-23T02:00:53Z","signature_id":"55d4b1840d69212b2703b61209abbc219f27e5b689e1fef7a10a1d77779d4060","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"55d4b1840d69212b2703b61209abbc219f27e5b689e1fef7a10a1d77779d4060","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"20cd25cfb84069bf0dbeae90ccfea853127cb3d7c1e8d8f537a39512ce31cc19","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"20cd25cfb84069bf0dbeae90ccfea853127cb3d7c1e8d8f537a39512ce31cc19","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"55d4b1840d69212b2703b61209abbc219f27e5b689e1fef7a10a1d77779d4060","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b936783c2d0b2a9d4ffabaeda14053cc9fef881b92be87e0e230002db2affbaf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":6,"runs_affected":5,"failed_runs":5,"run_impact_percent":19.23076923076923,"anchor_occurrences":6,"after_last_push_count":15,"also_in":["DEV","INT"],"trend_counts":[1,1,5,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":12,"prior_last_seen_at":"2026-08-19T23:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c244b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-v62dhvmn5xjx\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/433c7b9f-9ee6-405a-ac4b-61c2aba8b5b8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/433c7b9f-9ee6-405a-ac4b-61c2aba8b5b8\",\n      \"name\": \"433c7b9f-9ee6-405a-ac4b-61c2aba8b5b8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T23:23:38.105024916Z\",\n      \"endTime\": \"2026-08-19T23:47:40.969477415Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9ugi7ah3g0hdhfnrdfe02s4641un92 still exists (deletion dispatched at 2026-08-19T23:23:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c244b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-v62dhvmn5xjx\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/433c7b9f-9ee6-405a-ac4b-61c2aba8b5b8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/433c7b9f-9ee6-405a-ac4b-61c2aba8b5b8\",\n      \"name\": \"433c7b9f-9ee6-405a-ac4b-61c2aba8b5b8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T23:23:38.105024916Z\",\n      \"endTime\": \"2026-08-19T23:47:40.969477415Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9ugi7ah3g0hdhfnrdfe02s4641un92 still exists (deletion dispatched at 2026-08-19T23:23:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0013d62d0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-zk8j9gsrf47s\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83d822b3-ac2e-4cc4-966c-eab4ee8e8602\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83d822b3-ac2e-4cc4-966c-eab4ee8e8602\",\n      \"name\": \"83d822b3-ac2e-4cc4-966c-eab4ee8e8602\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T13:56:53.166820811Z\",\n      \"endTime\": \"2026-08-19T14:20:55.153951347Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9m491m1ahtrh178oqepopioqrgcdms still exists (deletion dispatched at 2026-08-19T13:56:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0013d62d0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-zk8j9gsrf47s\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83d822b3-ac2e-4cc4-966c-eab4ee8e8602\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83d822b3-ac2e-4cc4-966c-eab4ee8e8602\",\n      \"name\": \"83d822b3-ac2e-4cc4-966c-eab4ee8e8602\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T13:56:53.166820811Z\",\n      \"endTime\": \"2026-08-19T14:20:55.153951347Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9m491m1ahtrh178oqepopioqrgcdms still exists (deletion dispatched at 2026-08-19T13:56:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00112a408\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-rns7wq\" in resourcegroup=\"rg-zstream-upgrade-4-20-vhxh5b6zszlg\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1928d5f-397e-4b68-89ec-79ef6c7cd4ac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1928d5f-397e-4b68-89ec-79ef6c7cd4ac\",\n      \"name\": \"e1928d5f-397e-4b68-89ec-79ef6c7cd4ac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T14:13:37.892189859Z\",\n      \"endTime\": \"2026-08-19T14:37:44.990617474Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9mivqf2eovoc0ctilj3pgriaobnqh1 still exists (deletion dispatched at 2026-08-19T14:13:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00112a408\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-rns7wq\" in resourcegroup=\"rg-zstream-upgrade-4-20-vhxh5b6zszlg\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1928d5f-397e-4b68-89ec-79ef6c7cd4ac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1928d5f-397e-4b68-89ec-79ef6c7cd4ac\",\n      \"name\": \"e1928d5f-397e-4b68-89ec-79ef6c7cd4ac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T14:13:37.892189859Z\",\n      \"endTime\": \"2026-08-19T14:37:44.990617474Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9mivqf2eovoc0ctilj3pgriaobnqh1 still exists (deletion dispatched at 2026-08-19T14:13:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090212361167179776","occurred_at":"2026-08-19T23:00:07Z","signature_id":"31c34b662aa4007e0dd4d1ce9746adfafd06942edf2f371c9d1adf52e6ff6914","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090064577516539904","occurred_at":"2026-08-19T13:13:21Z","signature_id":"0b849c1b1930080096c80502fde35ce7acebfcf86d1a355be93bfad04a63ff81","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090064577516539904","occurred_at":"2026-08-19T13:13:21Z","signature_id":"9042ad9b49db684e855010022007c6f95a287cf403ee1d9dbdfb1c6014061aa8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089951420068401152","occurred_at":"2026-08-19T05:43:42Z","signature_id":"e5a5a72236358d122a4c716a2585811976fccb0e8b9b1d216e22cf2352d6eb1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089895148551933952","occurred_at":"2026-08-19T02:00:06Z","signature_id":"d0b9bf7336ba6c5e0c7514c62a1701c85355bf80d65b2d77b4cf498038933f28","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089849852383989760","occurred_at":"2026-08-18T23:00:06Z","signature_id":"f7b950d916f34a53d4d67029d02686a74ff69dcb54a3c16b19b71e8223f94bb5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3de4c85edc65d4851e95835c19adeaf11200c5ea73dde88d529944e306df4303","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":6,"runs_affected":5,"failed_runs":5,"run_impact_percent":19.23076923076923,"anchor_occurrences":6,"after_last_push_count":7,"also_in":["PROD"],"trend_counts":[0,0,3,2,0,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-20T08:35:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00118e480\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-20-h759ch\" in resourcegroup=\"rg-nightly-4-20-mzr4r5lz42qp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55ed743f-d106-4adb-b590-caa3369caeb3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55ed743f-d106-4adb-b590-caa3369caeb3\",\n      \"name\": \"55ed743f-d106-4adb-b590-caa3369caeb3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:25:48.884097042Z\",\n      \"endTime\": \"2026-08-24T23:49:50.709200875Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd7vm7aq4os1ksjghr3o1msj3ksno6l still exists (deletion dispatched at 2026-08-24T23:25:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00118e480\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-20-h759ch\" in resourcegroup=\"rg-nightly-4-20-mzr4r5lz42qp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55ed743f-d106-4adb-b590-caa3369caeb3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/55ed743f-d106-4adb-b590-caa3369caeb3\",\n      \"name\": \"55ed743f-d106-4adb-b590-caa3369caeb3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:25:48.884097042Z\",\n      \"endTime\": \"2026-08-24T23:49:50.709200875Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sd7vm7aq4os1ksjghr3o1msj3ksno6l still exists (deletion dispatched at 2026-08-24T23:25:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00111e108\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-95b2mn6x5nc7\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56a5601c-84db-4c4c-afe3-fa090d79a425\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56a5601c-84db-4c4c-afe3-fa090d79a425\",\n      \"name\": \"56a5601c-84db-4c4c-afe3-fa090d79a425\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:48:30.429013476Z\",\n      \"endTime\": \"2026-08-20T10:12:32.646924727Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7kb8vek727d6pbio7effbv09ei0ua still exists (deletion dispatched at 2026-08-20T09:48:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00111e108\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-95b2mn6x5nc7\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56a5601c-84db-4c4c-afe3-fa090d79a425\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56a5601c-84db-4c4c-afe3-fa090d79a425\",\n      \"name\": \"56a5601c-84db-4c4c-afe3-fa090d79a425\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:48:30.429013476Z\",\n      \"endTime\": \"2026-08-20T10:12:32.646924727Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7kb8vek727d6pbio7effbv09ei0ua still exists (deletion dispatched at 2026-08-20T09:48:31Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc002d34258\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-x729kk\" in resourcegroup=\"rg-candidate-4-22-6wvs8lznc8fp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\",\n      \"name\": \"bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:47:07.957787635Z\",\n      \"endTime\": \"2026-08-20T10:11:12.923142399Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7h5aduc2uvihvfe2vdijl20h32glr still exists (deletion dispatched at 2026-08-20T09:47:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc002d34258\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-x729kk\" in resourcegroup=\"rg-candidate-4-22-6wvs8lznc8fp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\",\n      \"name\": \"bdc947fc-d7e9-4fa7-ab9f-3f9b996c730c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:47:07.957787635Z\",\n      \"endTime\": \"2026-08-20T10:11:12.923142399Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sa7h5aduc2uvihvfe2vdijl20h32glr still exists (deletion dispatched at 2026-08-20T09:47:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"6a0460b71f8ba99286143da10d9b371325b39609e9612fa043b5980a861407e3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"3f5ac4cdd7ff19e9ff918b763fc044b6a568c83bba923b1245e9a1c9566d493d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"e0f6f930a179a3343bf2fd68ac2daa5ec05546590115b913738327ca0e47fe4d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090212361167179776","occurred_at":"2026-08-19T23:00:07Z","signature_id":"e605bb21ea5244e439cf62bcea1ffb88819d8256b4264429251651c7bd080039","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090159303129305088","occurred_at":"2026-08-19T19:29:45Z","signature_id":"0d48b115dcda077256c842a49dc3aa3b939af3e2623702d25a646ab55111bd97","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089990802548723712","occurred_at":"2026-08-19T08:20:12Z","signature_id":"1488dcf0a44d59096f0003c2bcb2ddab938abf19f850e64b113df3a7024a0e55","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"0914c49f679af2e655d272024a639b88ac1428a3970f243fbf49b919bf2490ff","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":6,"runs_affected":4,"failed_runs":4,"run_impact_percent":15.384615384615385,"anchor_occurrences":6,"after_last_push_count":146,"also_in":["DEV","PROD"],"trend_counts":[2,0,0,4,1,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":13,"prior_last_seen_at":"2026-08-21T23:00:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e8ea40\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-lf6jg45wtwxg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62db7451-86ec-4c6a-9227-bc775f891ebb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62db7451-86ec-4c6a-9227-bc775f891ebb\",\n      \"name\": \"62db7451-86ec-4c6a-9227-bc775f891ebb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:05:01.131009318Z\",\n      \"endTime\": \"2026-08-24T23:24:05.637766062Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e8ea40\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-lf6jg45wtwxg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62db7451-86ec-4c6a-9227-bc775f891ebb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/62db7451-86ec-4c6a-9227-bc775f891ebb\",\n      \"name\": \"62db7451-86ec-4c6a-9227-bc775f891ebb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:05:01.131009318Z\",\n      \"endTime\": \"2026-08-24T23:24:05.637766062Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f3f2e0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-sh428kn5d89n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3af2b0d-5fad-4125-9764-31b214f8fee1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3af2b0d-5fad-4125-9764-31b214f8fee1\",\n      \"name\": \"b3af2b0d-5fad-4125-9764-31b214f8fee1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T23:04:23.850321515Z\",\n      \"endTime\": \"2026-08-21T23:23:28.45210372Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f3f2e0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-sh428kn5d89n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3af2b0d-5fad-4125-9764-31b214f8fee1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3af2b0d-5fad-4125-9764-31b214f8fee1\",\n      \"name\": \"b3af2b0d-5fad-4125-9764-31b214f8fee1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T23:04:23.850321515Z\",\n      \"endTime\": \"2026-08-21T23:23:28.45210372Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-pt5scqn6n82h/cluster-candidate-4-21-28flhq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00124eb60\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-28flhq: failed waiting for cluster=\"cluster-candidate-4-21-28flhq\" in resourcegroup=\"rg-candidate-4-21-pt5scqn6n82h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/332f74a8-d0dc-4e4a-9737-70119dac90bc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/332f74a8-d0dc-4e4a-9737-70119dac90bc\",\n      \"name\": \"332f74a8-d0dc-4e4a-9737-70119dac90bc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T20:53:42.523371474Z\",\n      \"endTime\": \"2026-08-20T21:12:49.332578125Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-pt5scqn6n82h/cluster-candidate-4-21-28flhq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00124eb60\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-28flhq: failed waiting for cluster=\"cluster-candidate-4-21-28flhq\" in resourcegroup=\"rg-candidate-4-21-pt5scqn6n82h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/332f74a8-d0dc-4e4a-9737-70119dac90bc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/332f74a8-d0dc-4e4a-9737-70119dac90bc\",\n      \"name\": \"332f74a8-d0dc-4e4a-9737-70119dac90bc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T20:53:42.523371474Z\",\n      \"endTime\": \"2026-08-20T21:12:49.332578125Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"ce363ec1f1315846ac4ca18a663f8ca57451d8f5854815877a996fbdf3f45dd9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"fbf7d38b96f28f05b21778aab3674d4cce11c0c6f81909ef1d5dc73f06ad06e2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090538506672672768","occurred_at":"2026-08-20T20:36:34Z","signature_id":"75e8134db7c38c2272b0135b9ef67b8e4ba756be9b98292cf44f0135c9355207","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"56f9bbbf6c1cff5e3a43f7b42723ff5d2896274197334b06141ae6c0de7f3f12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"7ebf3a7566f82ffac1f2fdc94973962a97b9cebbccee3a25837ecc58e0903fb4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"f2f000bdc64541602873ebc887a909036d077a88b2e4d46c3fb4c2246d3285bd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b6258cd488b080bf5ba7099ce8387b9d2f4d222d926f9a0c81022dc736e89db2","failure_pattern":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","search_query":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":19.23076923076923,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,2,2,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:156]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0017be510\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:156]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0017be510\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:156]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001391020\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:156]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001391020\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:156]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013e8f30\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:156]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013e8f30\u003e: \n    VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091707289328160768","occurred_at":"2026-08-24T02:00:54Z","signature_id":"c14f9a5639d319819baa7a308cbb789519348e32b0abac6b02f5611b9187fbe7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"a4491e3bb9b86251a9e67c8b646d0760e0bef62043d0262fc0754c33958ef07f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2091344896731385856","occurred_at":"2026-08-23T02:00:53Z","signature_id":"9aa9e14981ec5eaa3b7eaab007d9e3c7d2f7fd49eb9bb6e90dc9b41d9aebe244","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"bd404f1ba751bc9877a1e8fdc24c7640aff9f6d2b3d0bdcdce835db61939593c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"417bf2d786aadfe3f04e5550d280ceae37c25a8bf717f073a992b1ac95f1cad4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"a146c4d50a72c43e4a69c21e18547ba5a2a0a2b53d6985079637bcec0f5acdda","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":5,"runs_affected":4,"failed_runs":4,"run_impact_percent":15.384615384615385,"anchor_occurrences":5,"after_last_push_count":11,"also_in":["DEV","INT","PROD"],"trend_counts":[0,3,2,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-08-03","2026-08-10","2026-08-17"],"prior_runs_affected":6,"prior_last_seen_at":"2026-08-19T23:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c54180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8hvpj6vtkqlf\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d19e078c-f7b8-4be7-ba0d-0921f4e3a585\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d19e078c-f7b8-4be7-ba0d-0921f4e3a585\",\n      \"name\": \"d19e078c-f7b8-4be7-ba0d-0921f4e3a585\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T23:47:16.516011942Z\",\n      \"endTime\": \"2026-08-20T00:11:22.445943521Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9uhn33u6bvucvlivpurd7qbjud4jvg still exists (deletion dispatched at 2026-08-19T23:47:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c54180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8hvpj6vtkqlf\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d19e078c-f7b8-4be7-ba0d-0921f4e3a585\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d19e078c-f7b8-4be7-ba0d-0921f4e3a585\",\n      \"name\": \"d19e078c-f7b8-4be7-ba0d-0921f4e3a585\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T23:47:16.516011942Z\",\n      \"endTime\": \"2026-08-20T00:11:22.445943521Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9uhn33u6bvucvlivpurd7qbjud4jvg still exists (deletion dispatched at 2026-08-19T23:47:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0017d6138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-ingress\" in resourcegroup=\"private-ingress-qxfhsjtcs5wr\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c2f0336-078f-4209-9e4d-a24e8e832089\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c2f0336-078f-4209-9e4d-a24e8e832089\",\n      \"name\": \"2c2f0336-078f-4209-9e4d-a24e8e832089\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:31:15.704278812Z\",\n      \"endTime\": \"2026-08-19T02:55:20.050088368Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9c23lo4u800am915f4oldars1ighvd still exists (deletion dispatched at 2026-08-19T02:31:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0017d6138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-ingress\" in resourcegroup=\"private-ingress-qxfhsjtcs5wr\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c2f0336-078f-4209-9e4d-a24e8e832089\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c2f0336-078f-4209-9e4d-a24e8e832089\",\n      \"name\": \"2c2f0336-078f-4209-9e4d-a24e8e832089\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T02:31:15.704278812Z\",\n      \"endTime\": \"2026-08-19T02:55:20.050088368Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9c23lo4u800am915f4oldars1ighvd still exists (deletion dispatched at 2026-08-19T02:31:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001aa82b8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-fr2tsbrfr7wk\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e32155a0-2078-4425-ba3d-ae626da965da\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e32155a0-2078-4425-ba3d-ae626da965da\",\n      \"name\": \"e32155a0-2078-4425-ba3d-ae626da965da\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T00:50:14.353170498Z\",\n      \"endTime\": \"2026-08-19T01:14:23.20531535Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9antrh42pdgmli6evu71asbu6o4q65 still exists (deletion dispatched at 2026-08-19T00:50:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001aa82b8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-fr2tsbrfr7wk\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e32155a0-2078-4425-ba3d-ae626da965da\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e32155a0-2078-4425-ba3d-ae626da965da\",\n      \"name\": \"e32155a0-2078-4425-ba3d-ae626da965da\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T00:50:14.353170498Z\",\n      \"endTime\": \"2026-08-19T01:14:23.20531535Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9antrh42pdgmli6evu71asbu6o4q65 still exists (deletion dispatched at 2026-08-19T00:50:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090212361167179776","occurred_at":"2026-08-19T23:00:07Z","signature_id":"b0ce1bdb91a35682a29122a0fcb2cafab1e41540901b85ad9cd097c5e00d92f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089895148551933952","occurred_at":"2026-08-19T02:00:06Z","signature_id":"c272e40d8f4e2729377a63b8f6a0cf17d1726c519963fbcdae85d0d65393f934","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089849852383989760","occurred_at":"2026-08-18T23:00:06Z","signature_id":"51a12a8ae1f8b970d58f63e5efd951b11da38c3ac85bd1f0e9c85fa4197f7be1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089849852383989760","occurred_at":"2026-08-18T23:00:06Z","signature_id":"98b90f81a27f8110105bfd382b1cc08266f8f295644da8fdd1d3d86d8f8d0989","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089760421908058112","occurred_at":"2026-08-18T17:04:45Z","signature_id":"3a6c6a873aa5da398e2d26070c62ec7a6a6fdf9873596c87a0624126608a1d5e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7e3c211098fa13aa2f965e23e8e58e902c8d4f3b340bbbf0b11eb0cf36bda7ed","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":15.384615384615385,"anchor_occurrences":4,"after_last_push_count":11,"also_in":["DEV","INT","PROD"],"trend_counts":[4,1,0,2,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-17"],"prior_runs_affected":10,"prior_last_seen_at":"2026-08-21T23:00:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:139]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006ae100\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-s82wq8qv974g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/389f219e-5f8d-4b6a-9c72-fdd66be0b32d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/389f219e-5f8d-4b6a-9c72-fdd66be0b32d\",\n      \"name\": \"389f219e-5f8d-4b6a-9c72-fdd66be0b32d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T23:17:03.253121618Z\",\n      \"endTime\": \"2026-08-21T23:36:09.200090899Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:139]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006ae100\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-s82wq8qv974g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/389f219e-5f8d-4b6a-9c72-fdd66be0b32d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/389f219e-5f8d-4b6a-9c72-fdd66be0b32d\",\n      \"name\": \"389f219e-5f8d-4b6a-9c72-fdd66be0b32d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T23:17:03.253121618Z\",\n      \"endTime\": \"2026-08-21T23:36:09.200090899Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:166]: failed to create node pool npupgrade-4-21 with version 4.21.28\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010bc7e0\u003e: \n    failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-8mw8p7\" in resourcegroup=\"rg-np-version-upgrade-8mw8p7-jzxbk7cd5ktl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/217c5b5b-876e-4dfd-aa27-a2f787a06b7d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/217c5b5b-876e-4dfd-aa27-a2f787a06b7d\",\n      \"name\": \"217c5b5b-876e-4dfd-aa27-a2f787a06b7d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:52:42.575458861Z\",\n      \"endTime\": \"2026-08-20T10:11:42.477430185Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:166]: failed to create node pool npupgrade-4-21 with version 4.21.28\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010bc7e0\u003e: \n    failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-8mw8p7\" in resourcegroup=\"rg-np-version-upgrade-8mw8p7-jzxbk7cd5ktl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/217c5b5b-876e-4dfd-aa27-a2f787a06b7d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/217c5b5b-876e-4dfd-aa27-a2f787a06b7d\",\n      \"name\": \"217c5b5b-876e-4dfd-aa27-a2f787a06b7d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:52:42.575458861Z\",\n      \"endTime\": \"2026-08-20T10:11:42.477430185Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009eaaa0\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-qjg88gwvq5xj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2facd99-ace4-452a-af16-9d4629a551d4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2facd99-ace4-452a-af16-9d4629a551d4\",\n      \"name\": \"c2facd99-ace4-452a-af16-9d4629a551d4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T04:35:11.139884226Z\",\n      \"endTime\": \"2026-08-20T04:54:12.658497794Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009eaaa0\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-qjg88gwvq5xj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2facd99-ace4-452a-af16-9d4629a551d4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2facd99-ace4-452a-af16-9d4629a551d4\",\n      \"name\": \"c2facd99-ace4-452a-af16-9d4629a551d4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T04:35:11.139884226Z\",\n      \"endTime\": \"2026-08-20T04:54:12.658497794Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"463a01fef05dd95fc4804e0b40a98b4138b6a654495626a1d5814204f1feb3ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"30e3ce44a70a32080f494a46962fb9e3504a602768cbec5e3cc893e94e06d32f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090276754840096768","occurred_at":"2026-08-20T03:16:28Z","signature_id":"e14f3f9d9bbacbaff6e558f257e54562c56eab915ed893444a9648f95512b4c2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2089645995238363136","occurred_at":"2026-08-18T09:30:03Z","signature_id":"7d14f1d5f95b8008bad06b36d2d49c4299e9113ff54678aee6c1f1dc3a63e616","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"c75b40567502299dd397c0248cce6097e2377df763d3deec7ae8c7f9cd09e3b2","failure_pattern":"node pool \u003cnodepool\u003e verification failed: \u003cnode\u003e (no version in containerRuntimeVersion \"cri-o://1","search_query":"node pool upgrade verification failed: g2a9g8l9y5u0b6x-nps-420-422-96vnb-jdgtr (no version in containerRuntimeVersion \"cri-o://1","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":15.384615384615385,"anchor_occurrences":4,"after_last_push_count":4,"also_in":["PROD"],"trend_counts":[0,0,0,0,1,1,1,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":4,"prior_last_seen_at":"2026-08-23T23:00:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0012c0d10\u003e: \n    node pool upgrade verification failed: w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb (no version in containerRuntimeVersion \"cri-o://1.35.4\"); w2k5s8r8i0t4x2o-nps-420-422-bqdkd-wx6fr (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]},{\"name\":\"w2k5s8r8i0t4x2o-nps-420-422-bqdkd-wx6fr\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc0012c0d10\u003e: \n    node pool upgrade verification failed: w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb (no version in containerRuntimeVersion \"cri-o://1.35.4\"); w2k5s8r8i0t4x2o-nps-420-422-bqdkd-wx6fr (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"w2k5s8r8i0t4x2o-nps-420-422-bqdkd-4jngb\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\"]},{\"name\":\"w2k5s8r8i0t4x2o-nps-420-422-bqdkd-wx6fr\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002a14f40\u003e: \n    node pool upgrade verification failed: x2t8p7p8m2s5y4s-nps-420-422-mqm7r-2pww5 (no version in containerRuntimeVersion \"cri-o://1.35.4\"); x2t8p7p8m2s5y4s-nps-420-422-mqm7r-vvlrc (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"x2t8p7p8m2s5y4s-nps-420-422-mqm7r-2pww5\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"x2t8p7p8m2s5y4s-nps-420-422-mqm7r-vvlrc\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002a14f40\u003e: \n    node pool upgrade verification failed: x2t8p7p8m2s5y4s-nps-420-422-mqm7r-2pww5 (no version in containerRuntimeVersion \"cri-o://1.35.4\"); x2t8p7p8m2s5y4s-nps-420-422-mqm7r-vvlrc (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"x2t8p7p8m2s5y4s-nps-420-422-mqm7r-2pww5\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"x2t8p7p8m2s5y4s-nps-420-422-mqm7r-vvlrc\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c6485404900b5004ca3a7e5aaa77e5c6b5bb7240121161d078bc15e9c3ef5162\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e3228f0dcb7a11d8eb20bb3c8b15da2d77a3fa8507223aae0c5777842a714728\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:458ccf68c0287122e5d22482719714b01b4ed8396787e1094786ec98449a2df0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:82c7829e1f6029dce1bef4743ca42327413d36214d9c40b09f8541dc68f2afad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9bf72765f0b6aba23570c1c09399e99a3c86a5160742943afd4d05e3db4bf9e7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7682214f2449957bd850f65d766bb182aced114511180df250066c89459018c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.005s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002bd0580\u003e: \n    node pool upgrade verification failed: g2a9g8l9y5u0b6x-nps-420-422-96vnb-jdgtr (no version in containerRuntimeVersion \"cri-o://1.35.4\"); g2a9g8l9y5u0b6x-nps-420-422-96vnb-tl58m (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"g2a9g8l9y5u0b6x-nps-420-422-96vnb-jdgtr\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b38493011dd1f654b1667e9c131cab5a11344a2ac32f414d457c0444559f12d1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bfa95dcd4a5aa349ba3fc73a0d653779cb753c344a53273cf05b9c32c7055da1\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d8c5e8a02e870bc7d4f63f9d441d972a1de4b77c816cda9252c13e6d604c3678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dbf8d9e6099a2ec5c27e70fc506279c744f9ea26fbe4a601d5dae168e174adf1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b2aeb5f663f8fef2b5de811d6159d00d166c5eba7d4251c98ea5b521ae07f00\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bcb807b6cee3e50ab1dc96db618f84b9d3963dc76507b63fe64db9dde3db5fd2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"g2a9g8l9y5u0b6x-nps-420-422-96vnb-tl58m\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b38493011dd1f654b1667e9c131cab5a11344a2ac32f414d457c0444559f12d1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bfa95dcd4a5aa349ba3fc73a0d653779cb753c344a53273cf05b9c32c7055da1\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d8c5e8a02e870bc7d4f63f9d441d972a1de4b77c816cda9252c13e6d604c3678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dbf8d9e6099a2ec5c27e70fc506279c744f9ea26fbe4a601d5dae168e174adf1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b2aeb5f663f8fef2b5de811d6159d00d166c5eba7d4251c98ea5b521ae07f00\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bcb807b6cee3e50ab1dc96db618f84b9d3963dc76507b63fe64db9dde3db5fd2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.005s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc002bd0580\u003e: \n    node pool upgrade verification failed: g2a9g8l9y5u0b6x-nps-420-422-96vnb-jdgtr (no version in containerRuntimeVersion \"cri-o://1.35.4\"); g2a9g8l9y5u0b6x-nps-420-422-96vnb-tl58m (no version in containerRuntimeVersion \"cri-o://1.35.4\"); nodes=[{\"name\":\"g2a9g8l9y5u0b6x-nps-420-422-96vnb-jdgtr\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b38493011dd1f654b1667e9c131cab5a11344a2ac32f414d457c0444559f12d1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bfa95dcd4a5aa349ba3fc73a0d653779cb753c344a53273cf05b9c32c7055da1\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20d78771136ba2e9a7aa535296eafe94ef196045f3a00a04a7f08d3fa6ad2d9a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6810e13d061f10e267e73c3fde34d10a89644114a034103cadaa4f5ad751571\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5cfa918ae07c4106e9cb91235788e607a140c3de42f02a0f81614092c0f5fb93\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:71083ff15d6aa4658fd679d03a905735a1a54fc49de0d72eab4e3366e498d41c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a605bd3607ba38f9904b3246eb04e657dadccce5359f5b76af74e0a72bb87cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd2343f595006f558a5b149f519d84bb3375a3103ff1ed4037830c502209d9d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:55bb6fc2dd45c4caaea16fb1f008eeb54f4b3b7a9ac852bcac140b61dd9dc85f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:afd2d3a5b70a05d98507e79c997593dcf9db357f16d3355a4868e632b5292719\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1d85150fcd72f4d601c093b5b2ba779da89c9ead8d4dd53b596c8b89d16ccc08\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8485f1881ece91e286057351277ac800f345a22e4d1bc1ab3ad7e73de17cef6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:144e0b078a15bb7fd714e4cf53069650721ff9fe2b73fd05921a961696721afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4292941aed2127039306505411bc6abf798085d7d15c0089231fec79671d624b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d8c5e8a02e870bc7d4f63f9d441d972a1de4b77c816cda9252c13e6d604c3678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dbf8d9e6099a2ec5c27e70fc506279c744f9ea26fbe4a601d5dae168e174adf1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b2aeb5f663f8fef2b5de811d6159d00d166c5eba7d4251c98ea5b521ae07f00\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bcb807b6cee3e50ab1dc96db618f84b9d3963dc76507b63fe64db9dde3db5fd2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:acabe3909ec92663ee0980963d10212c3e08e9f165eca2700f146aec32c7d1fa\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be9ba2395e6a8fd2b6c5afc4a6781a31c05f72b5679f77a3cf6ebc7556048e47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]},{\"name\":\"g2a9g8l9y5u0b6x-nps-420-422-96vnb-tl58m\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.35.4\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c2a780cb30d9032f2f6b40626e575dd48ecbd5808557398bf6c7baefda26ccc4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e69bac9d7618698a57aa3169948f353f3abe50ba67c5f041841ba4443657a5f1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:617996348f24e8651faa289768d9d01f2f855de9a9793b81d02c5cefc92ef1a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:856f008072cf2e9c50841f69b0dfba1703ab86a76b7a850290c5fc747fa57214\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0acb9aa82d2c0eb4394609dca6da3a950226fc73f10bd38df3bf2adfb053edf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3a693b8a10280fb9dec32eec3845cadb993134ee38afc2cb2890f87f0dc99111\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:270b2f636233a862bb97746e3b97c69320b499f53590b91ea9f6ca9798e6b95b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:29cd209b806f92660233e4b67f648ede22ddc20cba2c6a6bcbda992022830c52\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b38493011dd1f654b1667e9c131cab5a11344a2ac32f414d457c0444559f12d1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bfa95dcd4a5aa349ba3fc73a0d653779cb753c344a53273cf05b9c32c7055da1\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3ff0e77084bf0bd36460aa2532fe4a9ff67e0a50fdad3fa844646c2a956b476\",\"arohcpocpstg.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7bc458544a67245f4465b1de78924f32c827e4e53ab9a17cfa72f5be1f1a0ec\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:242c82f60fcf35b3c09f2de489d143ac4a1418bf6ab70f0b6d0959045c87604c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dfcb5d21731e8654ae9709ce20b0f1ad628b9a999cb72df8cc6142076cf6b22a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92ca67f742652a990f253f006c1503a8b6bf1684b22b7a2dd9b8339efa0bc42f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d3d4cc8de6d4b03dde4c806a8ada844bbbfb03f47b2f6bfc923d24f8c79efa57\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39035c7e2de8a140db90ec620f9f7d8d6b1ae9a5a4eae6238b02526aba4b44a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6206df7088a9ce0b19839198d49b75de875422f9b872c770013237cff1c1babd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ee6f3a63882ba5c2f7e761d9cdce292b55d0eb5ad199323d0ef1a07ddb08b08f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fc17e5a66f94564111e8ee4913d23ac45b0a034c61a5debe9fbce732f5677732\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:33ed2805042607c08b54e3040692fdb12763eaecaab721109c127e9a6e1e5648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b94009944719ea47e9ccb553ceee7c74a5983121c8658199012a6c0a09f3d147\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:35a7e61db3b3f21a10de8ad2c4df0f52212e29dc8839a50369efb90687f073f3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a30cd38d6591bbb0f81dc45b9face049a4fafa6bea60b020fb87cbcedfc8d060\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:143b03c645cbcc539b21d973918a3d4cabc1b5b97b285ab6bfa28ac87fdc7fe2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cb2c4564a9c0cb1a53c181cdb8c232b739bdcadf41e98103537b27809bb2306a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b8975696bfa7211144dd4b3519ad13dbcbe8ff793286cf5735203666fa55b56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cc662715c4f83ee4a129ddb533117bfaa95bea6dee22dad57a556bc1346092a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69128068c1b2145cb4424aa98ad3d4affed345eaf13216c71f6ee73fc6c702f0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:833efdc864a51200ee03e4f7857cb3f76c6ba8f82c8031a59518dd3b903891c7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8a31f34b179cb7c2420d1d7d703405e50777e1b4ee3deb714d22d9c379f76b64\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5682112322399b773223dc1ae7843b1eaafc4a3e0d136966c49eaa02e520d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ae8698744c915c26420510e9cc7939f22c85fd5647413606472d70025e7ba47\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8486fa29fdf815374bf127b860642bed575917c44a8d5cc3a82f1f14c3123e48\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39ea6ed4567969c3f1f69f2a24a196c7c877b5974d29c367154598774b674aa9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4330e3f6e6a1b7ddb2bc7136226d12e4f5aa495f66f2c53da15c354044c47b41\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64e7c1c2c1ebc3e7fd08a0ee88340c514a05b9731ba2319ddf56fe1b4c06ffd9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c45d3e850d9046d717b4f681f219afaa97170e0c343d38f4f89c7d8696a07ef2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c6f42dbedc645545c4210d86a0e85d51593f17587a66d30d1f056af06a0a59c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9cc52f6689e49349206f51e9fc55f9fd5b5cb0c25b445e5724016205384443b9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e2086907b8cd808dbdf49df4d6ff757c20d40883514462bd127760c277d6259\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6a093b6e3244495c45d8578af53693f36a40de3b53f84919d230d6fddf455f7c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a7161008f29e26910d7dda7e6c0c1886560317624c6879802975cc24261a6b97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f41d92c9bebe76fb811f271e648dde65a376496ce7eca1d125d01559835dfcb7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:01fa2753a35ec50f96664a523526680954ede835c5103e3ca1cdb0489c9d8f05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0df345275aa56086515f626638a554573328c22be5ff4ef1f74b1cc4d72d145f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4f0ce6dfeb2f2f82936f92a775108170e19f27680b313d1ca22934b53a2ff95a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cad9747010b5addc26808a1ca5ab8c0402da153b1aacc5dab65d55446f3815f8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:619cb3f003814da880a998260a4ffc7d1430c05acb15885e659cf9728039d13b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:79a712f6151d6c501567d4bcd140605b7e002aacd018d33065197c6ec73cf04e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4a7dcb90f64661e7e4d5bc63d9cfe4c1ebbdff4695926f13210fc7a1dabec1bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba0c482f1212af758f2d44a0d269d1045caff4f69905092bb621ce9f9a570dc7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:705dbb8f3d1207b33e04f5e2edb6167e517e50765e29558cea501b8c4bd5d355\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:768eb63513b08318e9c633094d60b1a0ecdedc93bfaf22592f4f9ac4e57eeacd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:392b0ef8c36398f9441d31f80e4b233a418cc77c9e3530d1052c3250f1933f55\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5a924e215fbd2e04c200b4bd9e2e7b9667160066ca9fe5731e1a2e100f30ed79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d8c5e8a02e870bc7d4f63f9d441d972a1de4b77c816cda9252c13e6d604c3678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dbf8d9e6099a2ec5c27e70fc506279c744f9ea26fbe4a601d5dae168e174adf1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3818f08ef58520bdbdbd668b304fdefd4f3952e4789849577a1f61ec5a29888c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ab8442cdb4ca07861755f7f98d983e60577cfc9c159daede9c96393c34c45a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0ecadce9bc3968cbdab409761e31fc3e9706b0983bc14c093252b18bf36580db\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:85cdf2b18e07f5cdd642a3d1552280637637e92761d23087c1d1d302545c6e05\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b2aeb5f663f8fef2b5de811d6159d00d166c5eba7d4251c98ea5b521ae07f00\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bcb807b6cee3e50ab1dc96db618f84b9d3963dc76507b63fe64db9dde3db5fd2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0031c188acf8712b7f7bd38f97ffaa55c72043f7fa4548c5e71f669bf43e25bf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:90d853bebbc4d60706b6bffcc0f080a7a1caad9d21cd45f249a4be59ade53c98\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69b1c16b5a1c98cfbc5fd38d3bb19ac93e5084b9b26a3e1ae3cee2349727a794\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:95efae9f45e4496d1451c96bcc0866412a971ca687530dccf81de35c40c1b31d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:038474ed33c60f474f40784c4c356c770fa1f854363444de831549bda3868fb6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5719d6a2898ab37f669911d025da8c3bc58d8395280d5e2e958407c30076897d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3dbfa79dcf3c51b06d821d56a1652d2c38f342c982305feba4f28df712088399\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e8f394246df3e18e3522fb9d5a660c5683a41d948bb28687061f237cc1d3a533\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ba932355c992fffc26d5422047521b6fcc613c78f675ec8ec37a69c6e20c98f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e85d556d9da5c3e0e855123fdd192da299ff6a19f53b86ba8e613515165fb519\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a86b8cb0138eb2cb1415ea1a2b6a30608b4957110629b0bf8dc8a4e846077bcf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e740fd5af3633855e8b36af5d5b0a0c6d1037ffe73030bd7558bfdc03428ab68\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1160c49464450d4f4c53de5f988656cea9f0c8729c7a042ed7d0f6377c1cd99d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6d4b8be8609b3debbe7c9efdccf84282581d94ae334e02f34945a10e86e70d68\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:03ff3448f5bb02f3a255eccd3c26730f65fdee658a5efb28ecad7e539bed9a8c\",\"arohcpocpstg.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1961efa7640ec88c399896e7a06e4f6f1c234e2d512eeb94c055980637d904e1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6c354101b4a3b9984b529150076bc671358c87a785fbf690a5c6dfab51173ae0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:08e92773fa95d4365a5e4e685742b2e1a10d4bb7e4f675f6a6e6e83b77a84f91\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31d154415a375b088aa2b3648b9c0fb0889aeb39089aa46395112a5306fcd212\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:310b2836a335755a8f429d3f2ab57c01d9cbd8b93912963d28d7a261534c16fb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44143796439f39eeaa08c6e8d2067c6f78ceebe481154267f89c4e9a7e960380\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:072f527c571a27e80b8a398e8a571de6b56659b1f964f76728c345e4d0807ad7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:36eabc5b1954f4c962bedca05ed5353fb0662b911d7fe548ac655f72073c461c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:875da3ddf07352f16bb090a9043d8a187f5b84c1f649bc20b2b48a8b00155bd5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:da2db623f1232a72ff4178da60ca3829fae53651a89d703c8ab6de9d5d3d1b73\"]}]\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"5565031e089c6eb68fe43eb98d2d43776a82305498e8c6a0a7f4c918b0348bf0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091661989980934144","occurred_at":"2026-08-23T23:00:54Z","signature_id":"6cb20baa379b7fcdcba1d14b4a55e6c86cfdee72a1c93591debfc4dfe3e0724b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2091299562508521472","occurred_at":"2026-08-22T23:00:44Z","signature_id":"749b338f0f666d9846dc55dd5356b693a9e5f5c64374dd55730a6f912a7a0323","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090937171316838400","occurred_at":"2026-08-21T23:00:43Z","signature_id":"9f5e26191b5a76b3738feae66a80838e9ebca5f7e61655121cd70c58e39a58ff","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"dc664af01d9bca7cbb1a073b6ebc6b372492bdba45274f79dc51fffb40636bd6","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":11.538461538461538,"anchor_occurrences":3,"after_last_push_count":3,"also_in":["DEV","PROD"],"trend_counts":[0,0,1,2,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-20T08:35:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000de13e0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-2kbvdf4gpb7z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ea7be1d-979f-43d0-a6eb-4fadf5b52158\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ea7be1d-979f-43d0-a6eb-4fadf5b52158\",\n      \"name\": \"1ea7be1d-979f-43d0-a6eb-4fadf5b52158\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:49:43.253613436Z\",\n      \"endTime\": \"2026-08-20T10:08:52.449510653Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000de13e0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-2kbvdf4gpb7z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ea7be1d-979f-43d0-a6eb-4fadf5b52158\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1ea7be1d-979f-43d0-a6eb-4fadf5b52158\",\n      \"name\": \"1ea7be1d-979f-43d0-a6eb-4fadf5b52158\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:49:43.253613436Z\",\n      \"endTime\": \"2026-08-20T10:08:52.449510653Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000678260\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-lvmrprf9szxn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc452150-a44c-4cdf-95f9-3ab1762bc659\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc452150-a44c-4cdf-95f9-3ab1762bc659\",\n      \"name\": \"dc452150-a44c-4cdf-95f9-3ab1762bc659\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T03:43:58.925008531Z\",\n      \"endTime\": \"2026-08-20T04:03:00.154367927Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000678260\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-lvmrprf9szxn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc452150-a44c-4cdf-95f9-3ab1762bc659\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc452150-a44c-4cdf-95f9-3ab1762bc659\",\n      \"name\": \"dc452150-a44c-4cdf-95f9-3ab1762bc659\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T03:43:58.925008531Z\",\n      \"endTime\": \"2026-08-20T04:03:00.154367927Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ff4e60\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-l88qjllvzfng\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c412073-0e2e-4d8b-bffc-e50b877db2c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c412073-0e2e-4d8b-bffc-e50b877db2c8\",\n      \"name\": \"3c412073-0e2e-4d8b-bffc-e50b877db2c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T14:26:29.558969897Z\",\n      \"endTime\": \"2026-08-19T14:45:35.6247326Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ff4e60\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-l88qjllvzfng\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c412073-0e2e-4d8b-bffc-e50b877db2c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c412073-0e2e-4d8b-bffc-e50b877db2c8\",\n      \"name\": \"3c412073-0e2e-4d8b-bffc-e50b877db2c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T14:26:29.558969897Z\",\n      \"endTime\": \"2026-08-19T14:45:35.6247326Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"1fe08479e59e5ed4afa0e1f161e992bf328632b651a998809d33c2ece866d07c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090276754840096768","occurred_at":"2026-08-20T03:16:28Z","signature_id":"7e2cf5d1bff2be054d231178fc2524db6c79b0da8fc042c6972e38c1b9001d27","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090064577516539904","occurred_at":"2026-08-19T13:13:21Z","signature_id":"a8c846c9bd4c317ad39a1011330ed9d1aa7ef3ae4b2a3224a56dbb4bb7373821","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"d58c4430f859260715a2df79338de096b1f7324f15367436040e29b8a0d491e6","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":7.6923076923076925,"anchor_occurrences":3,"after_last_push_count":6,"also_in":["DEV","INT","PROD"],"trend_counts":[0,0,0,0,1,2,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-03","2026-08-17"],"prior_runs_affected":5,"prior_last_seen_at":"2026-08-22T02:00:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000886800\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-7qtvldntpktk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ad306216-5622-4121-8bf1-f015261fa044\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ad306216-5622-4121-8bf1-f015261fa044\",\n      \"name\": \"ad306216-5622-4121-8bf1-f015261fa044\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T02:03:58.055302295Z\",\n      \"endTime\": \"2026-08-22T02:23:05.756072788Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000886800\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-7qtvldntpktk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ad306216-5622-4121-8bf1-f015261fa044\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ad306216-5622-4121-8bf1-f015261fa044\",\n      \"name\": \"ad306216-5622-4121-8bf1-f015261fa044\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T02:03:58.055302295Z\",\n      \"endTime\": \"2026-08-22T02:23:05.756072788Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000517020\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9zbjjzhbwsc7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8fbb6b40-a57a-44dc-9ea6-468eaf28a2fe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8fbb6b40-a57a-44dc-9ea6-468eaf28a2fe\",\n      \"name\": \"8fbb6b40-a57a-44dc-9ea6-468eaf28a2fe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T02:03:58.196286523Z\",\n      \"endTime\": \"2026-08-22T02:23:05.606457627Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000517020\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9zbjjzhbwsc7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8fbb6b40-a57a-44dc-9ea6-468eaf28a2fe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8fbb6b40-a57a-44dc-9ea6-468eaf28a2fe\",\n      \"name\": \"8fbb6b40-a57a-44dc-9ea6-468eaf28a2fe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-22T02:03:58.196286523Z\",\n      \"endTime\": \"2026-08-22T02:23:05.606457627Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008e61c0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gdl8mt5sbkqm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4841240d-e2f8-4813-8ee4-75401f65cda9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4841240d-e2f8-4813-8ee4-75401f65cda9\",\n      \"name\": \"4841240d-e2f8-4813-8ee4-75401f65cda9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:05:01.793828852Z\",\n      \"endTime\": \"2026-08-21T02:24:02.290641543Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008e61c0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gdl8mt5sbkqm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4841240d-e2f8-4813-8ee4-75401f65cda9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4841240d-e2f8-4813-8ee4-75401f65cda9\",\n      \"name\": \"4841240d-e2f8-4813-8ee4-75401f65cda9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:05:01.793828852Z\",\n      \"endTime\": \"2026-08-21T02:24:02.290641543Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"aa7e5f753e6e6e31752d40a0be892d0d8394c3a4ed7f539a58571438102dcd83","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"da606e988e3b163984f1144b1c2c0f76f91f7a639869fb04b4b28da971b69c0d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"90540ad1f71b81bed938e2d289c44e688fd651904bd2cb8cddef646e47d0d371","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"de0797a28411af3d24415a55d3d3897578493557d5bbf06af33b5687d5423fc9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":7.6923076923076925,"anchor_occurrences":2,"after_last_push_count":3,"also_in":["DEV","PROD"],"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-20T12:36:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-4r45c4hp7cql/cluster-candidate-4-20-r96znq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008fa700\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-r96znq: failed waiting for cluster=\"cluster-candidate-4-20-r96znq\" in resourcegroup=\"rg-candidate-4-20-4r45c4hp7cql\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3742f6f7-818c-4bbe-a0c4-ca70030eb73e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3742f6f7-818c-4bbe-a0c4-ca70030eb73e\",\n      \"name\": \"3742f6f7-818c-4bbe-a0c4-ca70030eb73e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:15:18.696546382Z\",\n      \"endTime\": \"2026-08-20T13:34:20.875186386Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-4r45c4hp7cql/cluster-candidate-4-20-r96znq should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008fa700\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-r96znq: failed waiting for cluster=\"cluster-candidate-4-20-r96znq\" in resourcegroup=\"rg-candidate-4-20-4r45c4hp7cql\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3742f6f7-818c-4bbe-a0c4-ca70030eb73e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3742f6f7-818c-4bbe-a0c4-ca70030eb73e\",\n      \"name\": \"3742f6f7-818c-4bbe-a0c4-ca70030eb73e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:15:18.696546382Z\",\n      \"endTime\": \"2026-08-20T13:34:20.875186386Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-gpmtwfgcr6xn/cluster-candidate-4-20-6mrg8j should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001090160\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-6mrg8j: failed waiting for cluster=\"cluster-candidate-4-20-6mrg8j\" in resourcegroup=\"rg-candidate-4-20-gpmtwfgcr6xn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1f59ecc-759d-4b6c-8b93-932b0208adf4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1f59ecc-759d-4b6c-8b93-932b0208adf4\",\n      \"name\": \"a1f59ecc-759d-4b6c-8b93-932b0208adf4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T08:53:33.518746975Z\",\n      \"endTime\": \"2026-08-20T09:12:39.900608486Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-gpmtwfgcr6xn/cluster-candidate-4-20-6mrg8j should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001090160\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-6mrg8j: failed waiting for cluster=\"cluster-candidate-4-20-6mrg8j\" in resourcegroup=\"rg-candidate-4-20-gpmtwfgcr6xn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1f59ecc-759d-4b6c-8b93-932b0208adf4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a1f59ecc-759d-4b6c-8b93-932b0208adf4\",\n      \"name\": \"a1f59ecc-759d-4b6c-8b93-932b0208adf4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T08:53:33.518746975Z\",\n      \"endTime\": \"2026-08-20T09:12:39.900608486Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","signature_id":"382e53c6f5440687f3efd2f0f735d848f4f2a75c27dbddc1aa370d48842e5ac7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"3aeb591de359ede5d82e500a19257072d85b9193e57ac8f4a13b125e2c1b0f69","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"81be2fb29cef27cb213a54edbaf774e087ffc3ae2e0885f48e9fd38eeccf1e0f","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: externalAuths, nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":7.6923076923076925,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["DEV"],"trend_counts":[0,1,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-19T13:13:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a4a120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-drrz4q8s78wp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ae99f150-7046-4373-b424-e51c79b56a22\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ae99f150-7046-4373-b424-e51c79b56a22\",\n      \"name\": \"ae99f150-7046-4373-b424-e51c79b56a22\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T14:30:35.01030213Z\",\n      \"endTime\": \"2026-08-19T14:54:35.627231464Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9mlei3q3ci7sc5kdghjha0ako2p2s2 still exists (deletion dispatched at 2026-08-19T14:30:35Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a4a120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-drrz4q8s78wp\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ae99f150-7046-4373-b424-e51c79b56a22\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ae99f150-7046-4373-b424-e51c79b56a22\",\n      \"name\": \"ae99f150-7046-4373-b424-e51c79b56a22\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T14:30:35.01030213Z\",\n      \"endTime\": \"2026-08-19T14:54:35.627231464Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9mlei3q3ci7sc5kdghjha0ako2p2s2 still exists (deletion dispatched at 2026-08-19T14:30:35Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011702b8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-dcntnkvt776v\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/948f6139-f188-494a-b1a0-5f0f1e06acaf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/948f6139-f188-494a-b1a0-5f0f1e06acaf\",\n      \"name\": \"948f6139-f188-494a-b1a0-5f0f1e06acaf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T00:47:39.330861834Z\",\n      \"endTime\": \"2026-08-19T01:11:43.18871778Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9ai28ppvdqu27hrgup8c3fclsl6ao7 still exists (deletion dispatched at 2026-08-19T00:47:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011702b8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-dcntnkvt776v\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/948f6139-f188-494a-b1a0-5f0f1e06acaf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/948f6139-f188-494a-b1a0-5f0f1e06acaf\",\n      \"name\": \"948f6139-f188-494a-b1a0-5f0f1e06acaf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T00:47:39.330861834Z\",\n      \"endTime\": \"2026-08-19T01:11:43.18871778Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s9ai28ppvdqu27hrgup8c3fclsl6ao7 still exists (deletion dispatched at 2026-08-19T00:47:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090064577516539904","occurred_at":"2026-08-19T13:13:21Z","signature_id":"e904a0dd0e231e642877fcf061d837be87b8253a6f6c4f6c048159da596c5ace","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2089849852383989760","occurred_at":"2026-08-18T23:00:06Z","signature_id":"e1b8b14d7b7e0ed37e674165acad3465a957c8cbcce8825b18083ed96f5b262f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3da6dfe5e2d468b0f7647b604914d8743369fe4db63b9b1f8f365f64663404b8","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":7.6923076923076925,"anchor_occurrences":2,"after_last_push_count":16,"also_in":["DEV","PROD"],"trend_counts":[2,1,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-10","2026-08-17"],"prior_runs_affected":17,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.005s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.005s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.006s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 1980.006s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"d06b61b2a290f31b2b44796608e3a50ed66dd83e5cff65620f4a4dbd778567f6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"2cfe09be092d341b2785b1ec7877c8436ce5f3a66545f57a16f2cc75e59ab47e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6fc45abab45bac910821b9bf649e0a8d2f5cae8a2886fe2e7857c4e0907eb987","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["DEV"],"trend_counts":[0,0,0,0,2,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-5jbtkc\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00147aa60\u003e: \n    failed to create HCP cluster np-noedge-5jbtkc: failed waiting for cluster=\"np-noedge-5jbtkc\" in resourcegroup=\"rg-np-noedge-5jbtkc-5kwghxcpjkf6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/edc3030e-e575-42e1-8662-b09b51002c15\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/edc3030e-e575-42e1-8662-b09b51002c15\",\n      \"name\": \"edc3030e-e575-42e1-8662-b09b51002c15\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:45:54.462061981Z\",\n      \"endTime\": \"2026-08-21T03:04:54.752602351Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, ingress-operator, cluster-image-registry-operator, catalog-operator, dns-operator, packageserver, redhat-operators-catalog, community-operators-catalog, cluster-version-operator, cluster-policy-controller, certified-operators-catalog, openshift-route-controller-manager, csi-snapshot-controller-operator, redhat-marketplace-catalog, cluster-network-operator, olm-operator, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:302]: failed to create HCP cluster np-noedge-5jbtkc\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00147aa60\u003e: \n    failed to create HCP cluster np-noedge-5jbtkc: failed waiting for cluster=\"np-noedge-5jbtkc\" in resourcegroup=\"rg-np-noedge-5jbtkc-5kwghxcpjkf6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/edc3030e-e575-42e1-8662-b09b51002c15\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/edc3030e-e575-42e1-8662-b09b51002c15\",\n      \"name\": \"edc3030e-e575-42e1-8662-b09b51002c15\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:45:54.462061981Z\",\n      \"endTime\": \"2026-08-21T03:04:54.752602351Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, ingress-operator, cluster-image-registry-operator, catalog-operator, dns-operator, packageserver, redhat-operators-catalog, community-operators-catalog, cluster-version-operator, cluster-policy-controller, certified-operators-catalog, openshift-route-controller-manager, csi-snapshot-controller-operator, redhat-marketplace-catalog, cluster-network-operator, olm-operator, hosted-cluster-config-operator, cluster-storage-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005fe2a0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-ttg7gz2lhlcp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ba59ea70-9b35-4783-a0ec-868e44666cde\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ba59ea70-9b35-4783-a0ec-868e44666cde\",\n      \"name\": \"ba59ea70-9b35-4783-a0ec-868e44666cde\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:50:23.471170015Z\",\n      \"endTime\": \"2026-08-21T03:09:24.769917396Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, redhat-operators-catalog, dns-operator, catalog-operator, cluster-image-registry-operator, community-operators-catalog, openshift-route-controller-manager, olm-operator, cluster-policy-controller, certified-operators-catalog, cluster-version-operator, openshift-controller-manager, cluster-node-tuning-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005fe2a0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-ttg7gz2lhlcp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ba59ea70-9b35-4783-a0ec-868e44666cde\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ba59ea70-9b35-4783-a0ec-868e44666cde\",\n      \"name\": \"ba59ea70-9b35-4783-a0ec-868e44666cde\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:50:23.471170015Z\",\n      \"endTime\": \"2026-08-21T03:09:24.769917396Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, redhat-operators-catalog, dns-operator, catalog-operator, cluster-image-registry-operator, community-operators-catalog, openshift-route-controller-manager, olm-operator, cluster-policy-controller, certified-operators-catalog, cluster-version-operator, openshift-controller-manager, cluster-node-tuning-operator, csi-snapshot-controller-operator, redhat-marketplace-catalog, hosted-cluster-config-operator, cluster-storage-operator, ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"2b76ccfe8aac8f29923c772019c68b4d1daa538e7e3fedec8fb0b2f71b8b486f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"91e88f33d22326cd1e1f28e5cf546d8a5f825f85b45e6974877ac8761f301bbc","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3b51c13a3cafe6196693705cdaea38d26216b0eac963a598136ad9cacb24071b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["DEV"],"trend_counts":[0,0,0,0,2,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-m6dffq9dg6px/cluster-candidate-4-22-rttbvz should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fee820\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-rttbvz: failed waiting for cluster=\"cluster-candidate-4-22-rttbvz\" in resourcegroup=\"rg-candidate-4-22-m6dffq9dg6px\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6621367e-101e-4d5a-ab22-9e61d95865dd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6621367e-101e-4d5a-ab22-9e61d95865dd\",\n      \"name\": \"6621367e-101e-4d5a-ab22-9e61d95865dd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:29:22.397457374Z\",\n      \"endTime\": \"2026-08-21T02:48:23.657141881Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-m6dffq9dg6px/cluster-candidate-4-22-rttbvz should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fee820\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-rttbvz: failed waiting for cluster=\"cluster-candidate-4-22-rttbvz\" in resourcegroup=\"rg-candidate-4-22-m6dffq9dg6px\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6621367e-101e-4d5a-ab22-9e61d95865dd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6621367e-101e-4d5a-ab22-9e61d95865dd\",\n      \"name\": \"6621367e-101e-4d5a-ab22-9e61d95865dd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:29:22.397457374Z\",\n      \"endTime\": \"2026-08-21T02:48:23.657141881Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-dqp9mm5cksqn/cluster-candidate-4-20-bfprqt should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015cf400\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-bfprqt: failed waiting for cluster=\"cluster-candidate-4-20-bfprqt\" in resourcegroup=\"rg-candidate-4-20-dqp9mm5cksqn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e423bc99-f4f6-4437-a051-cbb783aa9c19\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e423bc99-f4f6-4437-a051-cbb783aa9c19\",\n      \"name\": \"e423bc99-f4f6-4437-a051-cbb783aa9c19\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:18.115131874Z\",\n      \"endTime\": \"2026-08-21T02:22:22.541705225Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-dqp9mm5cksqn/cluster-candidate-4-20-bfprqt should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015cf400\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-bfprqt: failed waiting for cluster=\"cluster-candidate-4-20-bfprqt\" in resourcegroup=\"rg-candidate-4-20-dqp9mm5cksqn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e423bc99-f4f6-4437-a051-cbb783aa9c19\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e423bc99-f4f6-4437-a051-cbb783aa9c19\",\n      \"name\": \"e423bc99-f4f6-4437-a051-cbb783aa9c19\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:18.115131874Z\",\n      \"endTime\": \"2026-08-21T02:22:22.541705225Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"854ad62699d22398160556ff1d1461a048fa03f78571ec0dcf40580c9a2fd02d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"b3374a6a172ebb403ac686e50368e308458096eea2258fdb839e7b2281ad2231","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"bae2c31dfd388275f409339683e6035aa6871992170a42fd34eae18131b6e2ff","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["DEV","INT","PROD"],"trend_counts":[0,0,0,0,2,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-h762z2 with version 4.21.29\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000916f80\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-h762z2: failed waiting for cluster=\"np-version-upgrade-cluster-h762z2\" in resourcegroup=\"rg-np-version-upgrade-h762z2-227jm54s5fnw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a8fe4db-8a24-46bf-99be-8771ce79a814\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a8fe4db-8a24-46bf-99be-8771ce79a814\",\n      \"name\": \"7a8fe4db-8a24-46bf-99be-8771ce79a814\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:45:47.941159859Z\",\n      \"endTime\": \"2026-08-21T03:04:54.892784877Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-h762z2 with version 4.21.29\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000916f80\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-h762z2: failed waiting for cluster=\"np-version-upgrade-cluster-h762z2\" in resourcegroup=\"rg-np-version-upgrade-h762z2-227jm54s5fnw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a8fe4db-8a24-46bf-99be-8771ce79a814\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a8fe4db-8a24-46bf-99be-8771ce79a814\",\n      \"name\": \"7a8fe4db-8a24-46bf-99be-8771ce79a814\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:45:47.941159859Z\",\n      \"endTime\": \"2026-08-21T03:04:54.892784877Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c2e7a0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dhdl29xp87ff\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94b79bca-856d-47d2-b8ff-0c4f5c24fa7b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94b79bca-856d-47d2-b8ff-0c4f5c24fa7b\",\n      \"name\": \"94b79bca-856d-47d2-b8ff-0c4f5c24fa7b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:39:56.502029488Z\",\n      \"endTime\": \"2026-08-21T02:59:03.450785892Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c2e7a0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dhdl29xp87ff\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94b79bca-856d-47d2-b8ff-0c4f5c24fa7b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94b79bca-856d-47d2-b8ff-0c4f5c24fa7b\",\n      \"name\": \"94b79bca-856d-47d2-b8ff-0c4f5c24fa7b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:39:56.502029488Z\",\n      \"endTime\": \"2026-08-21T02:59:03.450785892Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"021ad8856d5c396fba67aba9c2c2ff60d51660b492523606a295b59e5f5dfb89","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"9b7bbad63103e55151b58e9fe0aa82ebcd0a55b9322b35d817145a7e77a3fb08","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"ab5d80437500581ad7ce3942572aca8a7c5ea9eda26f4367215f73be1501d7a0","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-l7xnjldf8s7s\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-l7xnjldf8s7s, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":2,"after_last_push_count":2,"also_in":["DEV"],"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:228]: failed to update node pool np-1 with new label and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002a1d1a0\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-l7xnjldf8s7s\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-l7xnjldf8s7s, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:228]: failed to update node pool np-1 with new label and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002a1d1a0\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-l7xnjldf8s7s\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-l7xnjldf8s7s, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:196]: failed to upgrade node pool npupgrade-4-20 to version 4.21.0-0.nightly-multi-2026-08-20-065923\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002d08a80\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-z5466s\" resourcegroup=\"rg-np-version-upgrade-z5466s-wkz757zpgbsr\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-z5466s in resource group rg-np-version-upgrade-z5466s-wkz757zpgbsr, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:196]: failed to upgrade node pool npupgrade-4-20 to version 4.21.0-0.nightly-multi-2026-08-20-065923\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002d08a80\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-z5466s\" resourcegroup=\"rg-np-version-upgrade-z5466s-wkz757zpgbsr\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-z5466s in resource group rg-np-version-upgrade-z5466s-wkz757zpgbsr, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"bef58880af3f67335fb38c9962fdc80b164c51531808fd3a944b58b5abc7835e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"fdab187cfe558b4bb3f41aa7470064acf21835a87bb5df9e4d6a394f761d1b45","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"f170a7927803930d63ef797276984cf62fcbd2f07de426245306d08da19e2bb9","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.005s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000b06150\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-08-20-052229, targetMinor=4.21.0-0.nightly-multi-2026-08-20-065923) failed: failed to get clusterversion \"version\": Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.005s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000b06150\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0-0.nightly-multi-2026-08-20-052229, targetMinor=4.21.0-0.nightly-multi-2026-08-20-065923) failed: failed to get clusterversion \"version\": Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.l4p1f9f5i1x6x0a.jubs.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.006s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f0c138\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22.0-0.nightly-multi-2026-08-20-141954, targetMinor=4.23.0-0.nightly-multi-2026-08-20-085020) failed: failed to get clusterversion \"version\": Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.006s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f0c138\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.22.0-0.nightly-multi-2026-08-20-141954, targetMinor=4.23.0-0.nightly-multi-2026-08-20-085020) failed: failed to get clusterversion \"version\": Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.p4k3x7x1w2y0n2w.c3ss.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"ee2ed8274c95a8959d110687ae9ece838bb447f44f7c22e5c06d8ad3a3a7f27b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"ee48f986fd9e592059d2a5cd09e88c38d974e8948f1e0f0065c5a8641bfef3cd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"d8e13d55daacf4ddf74bdc040314e449b308767e96d40780e45800466f812928","failure_pattern":"list nodes (nodePool=\u003cnodepool\u003e): Get \"\u003curl\u003e","search_query":"list nodes (nodePool=npdg-422-420): Get \"https://api","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:772]: Timed out after 2700.000s.\nnode pool nodes were not recreated at downgrade target version 4.20.0-0.nightly-multi-2026-08-20-052229\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000df20a0\u003e: \n    list nodes (nodePool=npdg-422-420): Get \"https://api.d1i3c9v1k0g4b5v.7gi9.uksouthstaging.aroapp-hcp.io:443/api/v1/nodes\": net/http: TLS handshake timeout\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:772]: Timed out after 2700.000s.\nnode pool nodes were not recreated at downgrade target version 4.20.0-0.nightly-multi-2026-08-20-052229\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000df20a0\u003e: \n    list nodes (nodePool=npdg-422-420): Get \"https://api.d1i3c9v1k0g4b5v.7gi9.uksouthstaging.aroapp-hcp.io:443/api/v1/nodes\": net/http: TLS handshake timeout\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0010e2920\u003e: \n    list nodes (nodePool=nps-420-422): Get \"https://api.m1w5a4x7b2n1l4i.t4j1.uksouthstaging.aroapp-hcp.io:443/api/v1/nodes\": EOF\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:493]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0010e2920\u003e: \n    list nodes (nodePool=nps-420-422): Get \"https://api.m1w5a4x7b2n1l4i.t4j1.uksouthstaging.aroapp-hcp.io:443/api/v1/nodes\": EOF\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"3519eaf3902036b2df340ad7c69439459fca96026f79aacac0e4ada2e8cb9ab4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"846c6382df9acef0fa5ddc2a6d1f4dc3f653f6d1f1033da3dc950698cda0412f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"44e2d333ce62abafbaf16d046b61703140a463730eb39f1d9ff95ea9875c0757","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":15,"also_in":["DEV","PROD"],"trend_counts":[2,1,0,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-10","2026-08-17"],"prior_runs_affected":16,"prior_last_seen_at":"2026-08-18T02:00:45Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2089532925895774208","occurred_at":"2026-08-18T02:00:45Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"d6ec87c7d972149dae5ab13de0394af8c0dc25683fd32e5ecca933f42f5cfc45","failure_pattern":"DeleteResourceGroup timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-z2b7swxgrmhl, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":2,"prior_window_starts":["2026-08-10","2026-08-17"],"prior_runs_affected":3,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:146]: failed to delete customer resource group \"cx-rg-cluster-z2b7swxgrmhl\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0039aa420\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-z2b7swxgrmhl, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:146]: failed to delete customer resource group \"cx-rg-cluster-z2b7swxgrmhl\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0039aa420\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-z2b7swxgrmhl, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"96a2fe2e929f73bfb9adc313d26af9eac1da35f5933f0823653985b4fdc7f529","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"90cdb36cc9a54567f53aaac3f8c671c2433ddd6b58320efddb4af3f213c50608","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","PROD"],"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011a6420\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-wpsgmw5rph44\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb23f88c-ec78-4c60-8fbf-d5847d914873\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb23f88c-ec78-4c60-8fbf-d5847d914873\",\n      \"name\": \"eb23f88c-ec78-4c60-8fbf-d5847d914873\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:05:11.172903386Z\",\n      \"endTime\": \"2026-08-21T02:24:12.542645954Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011a6420\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-wpsgmw5rph44\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb23f88c-ec78-4c60-8fbf-d5847d914873\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eb23f88c-ec78-4c60-8fbf-d5847d914873\",\n      \"name\": \"eb23f88c-ec78-4c60-8fbf-d5847d914873\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:05:11.172903386Z\",\n      \"endTime\": \"2026-08-21T02:24:12.542645954Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"7fb8dc9dd6caa75c82f132d73240ca1b73d2c2ed70d78cc29ea092d7fc3ce43b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6cef856251578d6d4f842b0ea51927280752396355aaa78eb5a3ffb310ef3d92","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager, kube-scheduler, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:140]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-52rvxg\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0018c2cc0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-52rvxg: failed waiting for cluster=\"cp-ystream-upgrade-4-22-52rvxg\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-sht9lcl99cmr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ab73772b-6b4b-4639-a6b8-b8a05b2d2e3a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ab73772b-6b4b-4639-a6b8-b8a05b2d2e3a\",\n      \"name\": \"ab73772b-6b4b-4639-a6b8-b8a05b2d2e3a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:33:22.674679438Z\",\n      \"endTime\": \"2026-08-21T02:52:23.424354393Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, cluster-storage-operator, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, kube-controller-manager, kube-scheduler, cluster-policy-controller, csi-snapshot-controller-operator, cluster-version-operator, openshift-route-controller-manager, catalog-operator, openshift-controller-manager, cluster-node-tuning-operator, dns-operator, certified-operators-catalog, hosted-cluster-config-operator, cluster-network-operator, olm-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:140]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-52rvxg\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0018c2cc0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-52rvxg: failed waiting for cluster=\"cp-ystream-upgrade-4-22-52rvxg\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-sht9lcl99cmr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ab73772b-6b4b-4639-a6b8-b8a05b2d2e3a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ab73772b-6b4b-4639-a6b8-b8a05b2d2e3a\",\n      \"name\": \"ab73772b-6b4b-4639-a6b8-b8a05b2d2e3a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:33:22.674679438Z\",\n      \"endTime\": \"2026-08-21T02:52:23.424354393Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, cluster-storage-operator, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog, community-operators-catalog, kube-controller-manager, kube-scheduler, cluster-policy-controller, csi-snapshot-controller-operator, cluster-version-operator, openshift-route-controller-manager, catalog-operator, openshift-controller-manager, cluster-node-tuning-operator, dns-operator, certified-operators-catalog, hosted-cluster-config-operator, cluster-network-operator, olm-operator, packageserver; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"a653dd0fe3070ac575df986a0df9a1327f00fa60786168762feb747e19bdb7f7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7356cf97f840631ab4a0a63ce4f36ca842474a0ca4160d56ce50ad927de67954","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, konnectivity-agent, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T08:35:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-sttt5r at version 4.22.10\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f130e0\u003e: \n    failed to create HCP cluster np-skip-minor-sttt5r: failed waiting for cluster=\"np-skip-minor-sttt5r\" in resourcegroup=\"rg-np-skip-minor-sttt5r-4fsvkfwk5vdh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a570b418-a849-489b-8b7e-8e8fb8135baa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a570b418-a849-489b-8b7e-8e8fb8135baa\",\n      \"name\": \"a570b418-a849-489b-8b7e-8e8fb8135baa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:36:05.972495231Z\",\n      \"endTime\": \"2026-08-20T09:55:11.507838131Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, azure-cloud-controller-manager, dns-operator, olm-operator, machine-approver, kube-controller-manager, hosted-cluster-config-operator, kube-storage-version-migrator, catalog-operator, olm-collect-profiles, kube-scheduler, community-operators-catalog, cluster-autoscaler, openshift-route-controller-manager, cluster-policy-controller, csi-snapshot-controller-operator, packageserver, control-plane-pki-operator, cluster-storage-operator, ingress-operator, konnectivity-agent, redhat-operators-catalog, openshift-apiserver, cluster-image-registry-operator, redhat-marketplace-catalog, certified-operators-catalog, openshift-controller-manager, cluster-network-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:442]: failed to create HCP cluster np-skip-minor-sttt5r at version 4.22.10\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f130e0\u003e: \n    failed to create HCP cluster np-skip-minor-sttt5r: failed waiting for cluster=\"np-skip-minor-sttt5r\" in resourcegroup=\"rg-np-skip-minor-sttt5r-4fsvkfwk5vdh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a570b418-a849-489b-8b7e-8e8fb8135baa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a570b418-a849-489b-8b7e-8e8fb8135baa\",\n      \"name\": \"a570b418-a849-489b-8b7e-8e8fb8135baa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:36:05.972495231Z\",\n      \"endTime\": \"2026-08-20T09:55:11.507838131Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, azure-cloud-controller-manager, dns-operator, olm-operator, machine-approver, kube-controller-manager, hosted-cluster-config-operator, kube-storage-version-migrator, catalog-operator, olm-collect-profiles, kube-scheduler, community-operators-catalog, cluster-autoscaler, openshift-route-controller-manager, cluster-policy-controller, csi-snapshot-controller-operator, packageserver, control-plane-pki-operator, cluster-storage-operator, ingress-operator, konnectivity-agent, redhat-operators-catalog, openshift-apiserver, cluster-image-registry-operator, redhat-marketplace-catalog, certified-operators-catalog, openshift-controller-manager, cluster-network-operator, cluster-node-tuning-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"50cf928a4b9ab91ad463923b6c69c34145bea77427cbfe46d31360d594882c1c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"17cbac65ad6dde679afa9f8374f5b421e45ae642715ea520e35f5231ee1e4b90","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, openshift-apiserver, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012312e0\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-mm9t5htdp974\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e464b7e0-c213-4e6e-988f-74f58142aa41\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e464b7e0-c213-4e6e-988f-74f58142aa41\",\n      \"name\": \"e464b7e0-c213-4e6e-988f-74f58142aa41\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:05:47.205015937Z\",\n      \"endTime\": \"2026-08-21T02:24:52.289709904Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, packageserver, redhat-marketplace-catalog, openshift-controller-manager, ingress-operator, olm-operator, redhat-operators-catalog, certified-operators-catalog, cluster-storage-operator, csi-snapshot-controller-operator, cluster-node-tuning-operator, hosted-cluster-config-operator, openshift-route-controller-manager, cluster-policy-controller, cluster-network-operator, catalog-operator, community-operators-catalog, dns-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012312e0\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-mm9t5htdp974\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e464b7e0-c213-4e6e-988f-74f58142aa41\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e464b7e0-c213-4e6e-988f-74f58142aa41\",\n      \"name\": \"e464b7e0-c213-4e6e-988f-74f58142aa41\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:05:47.205015937Z\",\n      \"endTime\": \"2026-08-21T02:24:52.289709904Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-version-operator, packageserver, redhat-marketplace-catalog, openshift-controller-manager, ingress-operator, olm-operator, redhat-operators-catalog, certified-operators-catalog, cluster-storage-operator, csi-snapshot-controller-operator, cluster-node-tuning-operator, hosted-cluster-config-operator, openshift-route-controller-manager, cluster-policy-controller, cluster-network-operator, catalog-operator, community-operators-catalog, dns-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"f4ef26d460058d90ed4ee949e2b53c1556150d27ed9ab6fc4a76d0e97cfd95ed","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"fcefb659555e6a990fedc929fc65e2b32862b237e18ebc38834dc9a0eaf288eb","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, konnectivity-agent, openshift-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dba740\u003e: \n    failed to create HCP cluster ea-listfncws6: failed waiting for cluster=\"ea-listfncws6\" in resourcegroup=\"ea-list-zhgflbxgws4b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04c8d3ea-1f82-496d-aa9a-800028c0f1a6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04c8d3ea-1f82-496d-aa9a-800028c0f1a6\",\n      \"name\": \"04c8d3ea-1f82-496d-aa9a-800028c0f1a6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:35:24.937663979Z\",\n      \"endTime\": \"2026-08-21T02:54:33.431541939Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-policy-controller, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dba740\u003e: \n    failed to create HCP cluster ea-listfncws6: failed waiting for cluster=\"ea-listfncws6\" in resourcegroup=\"ea-list-zhgflbxgws4b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04c8d3ea-1f82-496d-aa9a-800028c0f1a6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04c8d3ea-1f82-496d-aa9a-800028c0f1a6\",\n      \"name\": \"04c8d3ea-1f82-496d-aa9a-800028c0f1a6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:35:24.937663979Z\",\n      \"endTime\": \"2026-08-21T02:54:33.431541939Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-controller-manager, cluster-policy-controller, konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"5b3802d878773f1a0822d4136f1c861b8287fa1e3f87823166eff58535d9f1f9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"f969ddc66388f6636eb6fb562e07c82ce37d6c1699beba95e7c983471e1cd1bf","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller, kube-controller-manager, kube-scheduler, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0028eeae0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-92dczdzgqxgg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/739a7635-b32c-4f38-8193-2975abc5e092\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/739a7635-b32c-4f38-8193-2975abc5e092\",\n      \"name\": \"739a7635-b32c-4f38-8193-2975abc5e092\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:40:33.871209807Z\",\n      \"endTime\": \"2026-08-21T02:59:33.678476291Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, kube-scheduler; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0028eeae0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-92dczdzgqxgg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/739a7635-b32c-4f38-8193-2975abc5e092\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/739a7635-b32c-4f38-8193-2975abc5e092\",\n      \"name\": \"739a7635-b32c-4f38-8193-2975abc5e092\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:40:33.871209807Z\",\n      \"endTime\": \"2026-08-21T02:59:33.678476291Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-policy-controller, kube-scheduler; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"01d3c79c792237fb62672cad293b186ea02a3ea2191a33420c6fdd434b27d6f3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"84c9b6f5f7cc00976416d8874ac20f25a776a6f9140090905fa5d8c5a508047b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001344800\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-q78wj4tcrbbm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31dc1e1d-a696-4310-84a7-03cd3b7f433b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31dc1e1d-a696-4310-84a7-03cd3b7f433b\",\n      \"name\": \"31dc1e1d-a696-4310-84a7-03cd3b7f433b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:44:35.391627616Z\",\n      \"endTime\": \"2026-08-21T03:03:43.616320763Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001344800\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-q78wj4tcrbbm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31dc1e1d-a696-4310-84a7-03cd3b7f433b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31dc1e1d-a696-4310-84a7-03cd3b7f433b\",\n      \"name\": \"31dc1e1d-a696-4310-84a7-03cd3b7f433b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:44:35.391627616Z\",\n      \"endTime\": \"2026-08-21T03:03:43.616320763Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"8239e5a1956b5200678e34e906157e9a5f4c31e6442b526e2f03f3fb5593beb2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"96ac8f4694917334827139385bf0e43172c70aa0ffc95a888499b14a4f14ff9d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [konnectivity-agent, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001178800\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-njrg4blgvtnt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5481864c-7cad-43c3-99c6-7f83d731f12b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5481864c-7cad-43c3-99c6-7f83d731f12b\",\n      \"name\": \"5481864c-7cad-43c3-99c6-7f83d731f12b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:29.121350695Z\",\n      \"endTime\": \"2026-08-21T02:22:32.49197336Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [konnectivity-agent deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001178800\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-njrg4blgvtnt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5481864c-7cad-43c3-99c6-7f83d731f12b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5481864c-7cad-43c3-99c6-7f83d731f12b\",\n      \"name\": \"5481864c-7cad-43c3-99c6-7f83d731f12b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:03:29.121350695Z\",\n      \"endTime\": \"2026-08-21T02:22:32.49197336Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: konnectivity-agent; hosted cluster degraded: UnavailableReplicas: [konnectivity-agent deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"dc7a3b65992904f6bd95eec51bb94c8945852f4be0b05c741ee7f46e3bc1a620","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"a4c7965c412b351e80ef0544bff250a8a694f957336d67b40d995be9b67f72d9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":3,"also_in":["DEV"],"trend_counts":[2,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":2,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-stz97h\" with version 4.21.28 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005b4620\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-stz97h: failed waiting for cluster=\"cluster-zstream-4-21-stz97h\" in resourcegroup=\"rg-zstream-upgrade-4-21-jr9rvssp6jdl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc3c97f5-82ee-48a8-bb99-d6a0a63dc350\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc3c97f5-82ee-48a8-bb99-d6a0a63dc350\",\n      \"name\": \"dc3c97f5-82ee-48a8-bb99-d6a0a63dc350\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:32:46.784949272Z\",\n      \"endTime\": \"2026-08-21T02:51:53.64362497Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, certified-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-storage-operator, cluster-network-operator, ingress-operator, cluster-version-operator, cluster-policy-controller, dns-operator, cluster-image-registry-operator, csi-snapshot-controller-operator, catalog-operator, packageserver, olm-operator, redhat-operators-catalog, hosted-cluster-config-operator, cluster-node-tuning-operator, kube-controller-manager, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-stz97h\" with version 4.21.28 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005b4620\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-stz97h: failed waiting for cluster=\"cluster-zstream-4-21-stz97h\" in resourcegroup=\"rg-zstream-upgrade-4-21-jr9rvssp6jdl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc3c97f5-82ee-48a8-bb99-d6a0a63dc350\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc3c97f5-82ee-48a8-bb99-d6a0a63dc350\",\n      \"name\": \"dc3c97f5-82ee-48a8-bb99-d6a0a63dc350\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:32:46.784949272Z\",\n      \"endTime\": \"2026-08-21T02:51:53.64362497Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, certified-operators-catalog, openshift-controller-manager, openshift-route-controller-manager, cluster-storage-operator, cluster-network-operator, ingress-operator, cluster-version-operator, cluster-policy-controller, dns-operator, cluster-image-registry-operator, csi-snapshot-controller-operator, catalog-operator, packageserver, olm-operator, redhat-operators-catalog, hosted-cluster-config-operator, cluster-node-tuning-operator, kube-controller-manager, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-node-tuning-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 2 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"8bf1efa5a500673bb5d80c68b86c9d236a42c04862fb84b7d85434272615d259","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"c9f88b4bf47ba7b1ded7ca8c11c50ed78accd7528c6672b7fec47ef01c0992a8","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T13:13:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010f8e20\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-9qvbx4pls9rr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/668bdee1-fe77-422b-b467-4884f4f94057\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/668bdee1-fe77-422b-b467-4884f4f94057\",\n      \"name\": \"668bdee1-fe77-422b-b467-4884f4f94057\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T13:34:19.958633732Z\",\n      \"endTime\": \"2026-08-19T13:53:23.640928655Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010f8e20\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-9qvbx4pls9rr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/668bdee1-fe77-422b-b467-4884f4f94057\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/668bdee1-fe77-422b-b467-4884f4f94057\",\n      \"name\": \"668bdee1-fe77-422b-b467-4884f4f94057\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T13:34:19.958633732Z\",\n      \"endTime\": \"2026-08-19T13:53:23.640928655Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090064577516539904","occurred_at":"2026-08-19T13:13:21Z","signature_id":"fdd6fe2f501aa81c3def2e90227b41f195f175a3dc5f2d2da6aeb9b38455a514","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"80ca3cd678a9c2df1da684df660807a18dfa93c6de640958f03cc3d04dad8c8d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T13:13:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e9e8e0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wlkvs7868jnw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e159d376-391e-45cb-a50d-4792a4a230e8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e159d376-391e-45cb-a50d-4792a4a230e8\",\n      \"name\": \"e159d376-391e-45cb-a50d-4792a4a230e8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T13:32:05.123641839Z\",\n      \"endTime\": \"2026-08-19T13:51:13.689263263Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: hosted-cluster-config-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e9e8e0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wlkvs7868jnw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e159d376-391e-45cb-a50d-4792a4a230e8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e159d376-391e-45cb-a50d-4792a4a230e8\",\n      \"name\": \"e159d376-391e-45cb-a50d-4792a4a230e8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T13:32:05.123641839Z\",\n      \"endTime\": \"2026-08-19T13:51:13.689263263Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: hosted-cluster-config-operator deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090064577516539904","occurred_at":"2026-08-19T13:13:21Z","signature_id":"8fbb737fc21d5c791adf60078bc5daf0fea0e2cf41a6f52ad2aec57ec1f6dc65","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6722502d5d665731952f221ae4eff4c1fe60411f08b65ee3c8c1ea7fe37c4a2f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T12:36:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e68800\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rddwcsq6xrbg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf0b0684-dd10-4cf3-87c3-baf48258b7ce\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf0b0684-dd10-4cf3-87c3-baf48258b7ce\",\n      \"name\": \"bf0b0684-dd10-4cf3-87c3-baf48258b7ce\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:16:38.793143062Z\",\n      \"endTime\": \"2026-08-20T13:35:40.482871637Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e68800\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rddwcsq6xrbg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf0b0684-dd10-4cf3-87c3-baf48258b7ce\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bf0b0684-dd10-4cf3-87c3-baf48258b7ce\",\n      \"name\": \"bf0b0684-dd10-4cf3-87c3-baf48258b7ce\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T13:16:38.793143062Z\",\n      \"endTime\": \"2026-08-20T13:35:40.482871637Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090419881110409216","occurred_at":"2026-08-20T12:36:44Z","signature_id":"37da2bd93766d6d4d4f6961c9e8500becdfb67c804e717112af85f0368264347","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"c6b1360d9b766b9ab9834bb704955b4878d98ef788b91a435d90ca99b95a962b","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \"\u003curl\u003e\": EOF; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, machine-approver, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T13:13:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000806660\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-2c57lt79tp2l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e60c22e9-c32c-4219-b8d7-30131e20f6f1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e60c22e9-c32c-4219-b8d7-30131e20f6f1\",\n      \"name\": \"e60c22e9-c32c-4219-b8d7-30131e20f6f1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T13:33:35.958939424Z\",\n      \"endTime\": \"2026-08-19T13:52:43.668779021Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.m3b8f2l5y3g1d7d.320w.uksouthstaging.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000806660\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-2c57lt79tp2l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e60c22e9-c32c-4219-b8d7-30131e20f6f1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e60c22e9-c32c-4219-b8d7-30131e20f6f1\",\n      \"name\": \"e60c22e9-c32c-4219-b8d7-30131e20f6f1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T13:33:35.958939424Z\",\n      \"endTime\": \"2026-08-19T13:52:43.668779021Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.m3b8f2l5y3g1d7d.320w.uksouthstaging.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090064577516539904","occurred_at":"2026-08-19T13:13:21Z","signature_id":"d68e326b70076786b8e9b9f228d830ccd3f5c6fff7bafb198fa23e312a38b993","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"4b6bc70dde3370d5e666a11dcdf8b092a80443f417ead980036aeb5b45355b1f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \"\u003curl\u003e\": EOF; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog, ignition-server, redhat-marketplace-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T08:35:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7af60\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-kfjt8dvwt56j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/54f121a6-620a-4f93-bac1-3e84405aac12\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/54f121a6-620a-4f93-bac1-3e84405aac12\",\n      \"name\": \"54f121a6-620a-4f93-bac1-3e84405aac12\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:44:43.862276988Z\",\n      \"endTime\": \"2026-08-20T10:03:52.490075448Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.a8j1x1l8p8y5l8a.lecf.uksouthstaging.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7af60\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-kfjt8dvwt56j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/54f121a6-620a-4f93-bac1-3e84405aac12\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/54f121a6-620a-4f93-bac1-3e84405aac12\",\n      \"name\": \"54f121a6-620a-4f93-bac1-3e84405aac12\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-20T09:44:43.862276988Z\",\n      \"endTime\": \"2026-08-20T10:03:52.490075448Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.a8j1x1l8p8y5l8a.lecf.uksouthstaging.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2090356957637513216","occurred_at":"2026-08-20T08:35:04Z","signature_id":"84a396fab7c2ab3ae1588023933d5e474ab7b143128d353980cd15000536f48b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"dc30c2cd1479d681b3c8e85dbe87f378840f2bf26b28373ee01d7b48d759e41c","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \"\u003curl\u003e\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server, konnectivity-agent, kube-scheduler]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-5lt48h with version 4.21.29\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001068a60\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-5lt48h: failed waiting for cluster=\"np-version-upgrade-cluster-5lt48h\" in resourcegroup=\"rg-np-version-upgrade-5lt48h-dszsrsbrq6zk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe765ae8-0a48-4400-a1c0-66c693e2b5fa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe765ae8-0a48-4400-a1c0-66c693e2b5fa\",\n      \"name\": \"fe765ae8-0a48-4400-a1c0-66c693e2b5fa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:45:36.662949608Z\",\n      \"endTime\": \"2026-08-21T03:04:44.745401761Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.q5e3m9a0x8h7s0u.7eq2.uksouthstaging.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:147]: failed to create HCP cluster np-version-upgrade-cluster-5lt48h with version 4.21.29\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001068a60\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-5lt48h: failed waiting for cluster=\"np-version-upgrade-cluster-5lt48h\" in resourcegroup=\"rg-np-version-upgrade-5lt48h-dszsrsbrq6zk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe765ae8-0a48-4400-a1c0-66c693e2b5fa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe765ae8-0a48-4400-a1c0-66c693e2b5fa\",\n      \"name\": \"fe765ae8-0a48-4400-a1c0-66c693e2b5fa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:45:36.662949608Z\",\n      \"endTime\": \"2026-08-21T03:04:44.745401761Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.q5e3m9a0x8h7s0u.7eq2.uksouthstaging.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [ignition-server deployment has 1 unavailable replicas, konnectivity-agent deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"ff8b79ab35f4ee1da283d000f55c6cb2c5767888f63ecfe86c10a751852965b5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"c6e3b20deb9bae580ddd8091aa6a577e844f6b83692c9312152315f89fed55d3","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, cluster-api]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ce5680\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-s6nq8d7z6854\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68d2c327-4b78-47bd-bbdb-95c6f476f1ab\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68d2c327-4b78-47bd-bbdb-95c6f476f1ab\",\n      \"name\": \"68d2c327-4b78-47bd-bbdb-95c6f476f1ab\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:04:33.015430331Z\",\n      \"endTime\": \"2026-08-21T02:23:42.504347676Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ce5680\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-s6nq8d7z6854\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68d2c327-4b78-47bd-bbdb-95c6f476f1ab\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/68d2c327-4b78-47bd-bbdb-95c6f476f1ab\",\n      \"name\": \"68d2c327-4b78-47bd-bbdb-95c6f476f1ab\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:04:33.015430331Z\",\n      \"endTime\": \"2026-08-21T02:23:42.504347676Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, cluster-api deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"60abba32a30fc314bf3f1591fe14612458a80c23ca459244b846d1098cad1d5d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"c94f79fadf5f0c5900ad5aba81751610a3b4c431f48765be0c4ce0b0f37bd7e9","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-21T02:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001048e40\u003e: \n    failed to create HCP cluster np-dg-minor-dc29s7: failed waiting for cluster=\"np-dg-minor-dc29s7\" in resourcegroup=\"rg-np-dg-minor-dc29s7-9gmklftjh28x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6bbe4242-1a21-482f-8b6d-fe7e97ed8cb2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6bbe4242-1a21-482f-8b6d-fe7e97ed8cb2\",\n      \"name\": \"6bbe4242-1a21-482f-8b6d-fe7e97ed8cb2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:49:33.691417488Z\",\n      \"endTime\": \"2026-08-21T03:08:34.727253217Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:721]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001048e40\u003e: \n    failed to create HCP cluster np-dg-minor-dc29s7: failed waiting for cluster=\"np-dg-minor-dc29s7\" in resourcegroup=\"rg-np-dg-minor-dc29s7-9gmklftjh28x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6bbe4242-1a21-482f-8b6d-fe7e97ed8cb2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6bbe4242-1a21-482f-8b6d-fe7e97ed8cb2\",\n      \"name\": \"6bbe4242-1a21-482f-8b6d-fe7e97ed8cb2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-21T02:49:33.691417488Z\",\n      \"endTime\": \"2026-08-21T03:08:34.727253217Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090619934307520512","occurred_at":"2026-08-21T02:00:08Z","signature_id":"7ddd8e0bddb95c4a8ef63e85e6c7889f6f625f289311cf197d6695415a140810","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"db86d0a7b49ef93daf7cdb63d12e83b79ddcd71210e2847a436b8d80dd5dde7f","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] \u003cno_message\u003e; [clusterServiceStatus] \u003cno_message\u003e; [descendantResources] \u003cno_message\u003e; [hostedCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001716270\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-nml9dcvzzfr9\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e8f4e5-f6d1-44fe-818c-64543efd65cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e8f4e5-f6d1-44fe-818c-64543efd65cf\",\n      \"name\": \"d0e8f4e5-f6d1-44fe-818c-64543efd65cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:14:11.789973414Z\",\n      \"endTime\": \"2026-08-24T23:38:15.678890881Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001716270\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-nml9dcvzzfr9\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e8f4e5-f6d1-44fe-818c-64543efd65cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e8f4e5-f6d1-44fe-818c-64543efd65cf\",\n      \"name\": \"d0e8f4e5-f6d1-44fe-818c-64543efd65cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-24T23:14:11.789973414Z\",\n      \"endTime\": \"2026-08-24T23:38:15.678890881Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] \\u003cno_message\\u003e; [clusterServiceStatus] \\u003cno_message\\u003e; [descendantResources] \\u003cno_message\\u003e; [hostedCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2092024382493822976","occurred_at":"2026-08-24T23:00:55Z","signature_id":"3feed0e49b703d23a4823f602dec7fe62c46bf78411b06f90b5ee2c2bde72da5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"ee72f6e8217d23c4c37addc0dee5930c5b90fe95a9029df3ed16ce9a3769fd8f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message [controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-17-010411 to 4.23.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-19T23:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:174]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-w8t8gk\" to 4.23.0-0.nightly-multi-2026-08-19-140455\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00160f3a0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-w8t8gk\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-ldgb8b99w4xp\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/97cf82cb-bb1f-4aac-aaea-96415f8aa42a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/97cf82cb-bb1f-4aac-aaea-96415f8aa42a\",\n      \"name\": \"97cf82cb-bb1f-4aac-aaea-96415f8aa42a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T23:45:11.57498354Z\",\n      \"endTime\": \"2026-08-19T23:45:21.100010011Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-17-010411 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:174]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-w8t8gk\" to 4.23.0-0.nightly-multi-2026-08-19-140455\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00160f3a0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-w8t8gk\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-ldgb8b99w4xp\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/97cf82cb-bb1f-4aac-aaea-96415f8aa42a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/97cf82cb-bb1f-4aac-aaea-96415f8aa42a\",\n      \"name\": \"97cf82cb-bb1f-4aac-aaea-96415f8aa42a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-19T23:45:11.57498354Z\",\n      \"endTime\": \"2026-08-19T23:45:21.100010011Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"[controlPlaneDesiredVersionResolution] no upgrade path found from 4.22.0-0.nightly-multi-2026-08-17-010411 to 4.23.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090212361167179776","occurred_at":"2026-08-19T23:00:07Z","signature_id":"337a8761f22e379b1943e8bf13e36bffe4d94278c9dc5ca3fef8e2f8362ca41d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"8dad9a32124d40126133bba2520cede07e7f2a6cd0df33b90c1e690aeb9f3131","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.20\": must be at least 4.20.34; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T02:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-4lxbwp\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0011b1e00\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-p9w9mgdh26kn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-4lxbwp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.33\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.34\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-20-4lxbwp\" to minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0011b1e00\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-p9w9mgdh26kn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-4lxbwp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": may not decrease from 4.20.33\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.20\\\": must be at least 4.20.34\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090257540762832896","occurred_at":"2026-08-20T02:00:07Z","signature_id":"4ebee03d5cb8560c61e868d8bbb831f73868f90ba39c7b26a29c917e8892dc28","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7ce22653c1790596420acf5ec30dfcb3f90e95095df0cfef47397e47b57f8997","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21\": must be at least 4.21.29; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T02:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-qkbdvr\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008bcae0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qdqdx9dwcprj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-qkbdvr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.28\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.29\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-21-qkbdvr\" to minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008bcae0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qdqdx9dwcprj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-qkbdvr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": may not decrease from 4.21.28\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21\\\": must be at least 4.21.29\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090257540762832896","occurred_at":"2026-08-20T02:00:07Z","signature_id":"8e749cbda0987bd36e3427fb17c74e6efb4a7ffd7f85d804ab73f254f7c2b973","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"4a4f87d4133fd3a9df163a91b47668459c543343ba1d123ba27a73ceaa3385ce","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.22\": must be at least 4.22.9; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T02:00:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-fnbgn2\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0011309f0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-cbhdqjqgzmjz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-fnbgn2\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.9\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.9\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:148]: failed to pin cluster \"cluster-zstream-4-22-fnbgn2\" to minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0011309f0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-cbhdqjqgzmjz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-fnbgn2\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": may not decrease from 4.22.9\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.22\\\": must be at least 4.22.9\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090257540762832896","occurred_at":"2026-08-20T02:00:07Z","signature_id":"2e88e71b6a16994e8d9342a6dfffd6392efeb37bdd6569b14205b6f91d0d4a24","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"0b5ec0cbb59802a88f2bc42deba73fa247a353c118d64efe8e80ca88b66f3f5b","failure_pattern":"Test Panicked","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:475]: Test Panicked","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-22T02:00:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["github.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:475\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0xc001716000?, {0x7f5ef448fe78?, 0xc000f7dd10?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:1039 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0xc000aceb00, {0x7f5ef448fe78, 0xc000f7dd10}, {0xc001436078, 0x17}, {0xc0000140b9, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:464 +0x2a9\ngithub.com/Azure/ARO-HCP/test/e2e.init.func26.1({0x7f5ef448fe78, 0xc000f7dd10}, {0xeb544f6, 0x4})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:88 +0x6a5\nreflect.Value.call({0xc477720?, 0xf1650a0?, 0x13?}, {0xeb54646, 0x4}, {0xc000f7dda0, 0x2, 0x2?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcca\nreflect.Value.Call({0xc477720?, 0xf1650a0?, 0xfc03900?}, {0xc000f7dda0?, 0x20?, 0xc755ac0?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9\ngithub.com/onsi/ginkgo/v2.invokeFunction({0xc477720, 0xf1650a0}, {0xc000f5d6e0, 0x2, 0xc755ac0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:339 +0x2c5\ngithub.com/onsi/ginkgo/v2.generateTable.func2.1({0xfb6b1a0?, 0xc000f7dd10?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:293 +0x105\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 101\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:475]: Test Panicked: failed to get subscription ID: subscription with name 'XXXXXXXXXXX Hosted Clusters - Stage - 00' not found\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:475]: Test Panicked\ngithub.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:475\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0xc001716000?, {0x7f5ef448fe78?, 0xc000f7dd10?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:1039 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0xc000aceb00, {0x7f5ef448fe78, 0xc000f7dd10}, {0xc001436078, 0x17}, {0xc0000140b9, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:464 +0x2a9\ngithub.com/Azure/ARO-HCP/test/e2e.init.func26.1({0x7f5ef448fe78, 0xc000f7dd10}, {0xeb544f6, 0x4})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:88 +0x6a5\nreflect.Value.call({0xc477720?, 0xf1650a0?, 0x13?}, {0xeb54646, 0x4}, {0xc000f7dda0, 0x2, 0x2?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcca\nreflect.Value.Call({0xc477720?, 0xf1650a0?, 0xfc03900?}, {0xc000f7dda0?, 0x20?, 0xc755ac0?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9\ngithub.com/onsi/ginkgo/v2.invokeFunction({0xc477720, 0xf1650a0}, {0xc000f5d6e0, 0x2, 0xc755ac0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:339 +0x2c5\ngithub.com/onsi/ginkgo/v2.generateTable.func2.1({0xfb6b1a0?, 0xc000f7dd10?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:293 +0x105\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 101\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:475]: Test Panicked: failed to get subscription ID: subscription with name 'XXXXXXXXXXX Hosted Clusters - Stage - 00' not found\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:475]: Test Panicked"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2090982470185914368","occurred_at":"2026-08-22T02:00:44Z","signature_id":"8a15fba718b6d75d8d44dad039889bf9f5ba45b37ef0458c98c5fa51b3f5ce6d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b748a3b1c1b9c7fd8f7f617ddf190f4c65999a469c2f67349a1718ac3211e788","failure_pattern":"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5","search_query":"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0013e8a98\u003e: \n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-08-20-024947, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: failed to get clusterversion \"version\": Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": EOF\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": EOF\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/version\": EOF\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": EOF\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0013e8a98\u003e: \n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-08-20-024947, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: failed to get clusterversion \"version\": Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": EOF\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": EOF\n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/version\": EOF\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.v7t4t4y8f3t0a2n.l1x0.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": EOF\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"9597e11d65e1b6b9e701068b23031343892ee3c2b207fcc034b59895dd98d0c1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"bd4fdaffaf3b15f286ad5f2ca30cb95f7541a63bca5896bf32cc0ecd46a82996","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"\u003curl\u003e","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.8461538461538463,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-08-17..2026-08-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-08-17"],"prior_runs_affected":1,"prior_last_seen_at":"2026-08-20T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000fcc5b8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-20-065923, targetMinor=4.22.0-0.nightly-multi-2026-08-20-141954) failed: failed to get clusterversion \"version\": Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:183]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000fcc5b8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: get kube-apiserver ServerVersion: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/version\": net/http: TLS handshake timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": net/http: TLS handshake timeout\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-08-20-065923, targetMinor=4.22.0-0.nightly-multi-2026-08-20-141954) failed: failed to get clusterversion \"version\": Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.u0i9f7m5k4k1z6j.gvbd.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": net/http: TLS handshake timeout\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2090574634805104640","occurred_at":"2026-08-20T23:00:08Z","signature_id":"28d3b5e4d6e1e3b6c0885d78d546ff6d719fb6e7e972de708efbba62c246ae05","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}