fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:193]: failed to verify simple web app runs on cluster "cluster-candidate-4-23-b9bhxn"
Unexpected error:
    <*fmt.wrapError | 0x83317de340>: 
    DNS for route host agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x8331542d60>{
            msg: "DNS for agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:193]: failed to verify simple web app runs on cluster "cluster-candidate-4-23-b9bhxn"
Unexpected error:
    <*fmt.wrapError | 0x83317de340>: 
    DNS for route host agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0x8331542d60>{
            msg: "DNS for agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-r747r.apps.aro.v9m6n2u3u2g4s3m.5dfw.j0466048.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred

time=2026-10-22T23:38:58.792Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 description="Step deploy\n Kind: Helm\n"
time=2026-10-22T23:38:59.096Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 values="map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:b98f9f1c3c603f0e3ee70cd3d70746d8265b8d425df5b71f23c4b0dcb3eb979c imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:b451542dd17a9b1b1b400fa1ca768a492720b87b696f373f0494217d855aee8c imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:<nil>]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]"
time=2026-10-22T23:38:59.096Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2
time=2026-10-22T23:38:59.335Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 namespace=secret-sync-controller
time=2026-10-22T23:38:59.485Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 dryRun=false
time=2026-10-22T23:38:59.485Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2
time=2026-10-22T23:38:59.547Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2
time=2026-10-22T23:44:03.236Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err="resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\ncontext deadline exceeded"
time=2026-10-22T23:44:03.237Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err="stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\ncontext deadline exceeded"

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: HCP cluster rg-candidate-4-23-h5dw5xj4gsz5/cluster-candidate-4-23-q8dfp4 should provision
Unexpected error:
    <*fmt.wrapError | 0x202fbf30a20>: 
    failed to create HCP cluster cluster-candidate-4-23-q8dfp4: failed waiting for cluster="cluster-candidate-4-23-q8dfp4" in resourcegroup="rg-candidate-4-23-h5dw5xj4gsz5" to finish creating: GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1daf02db-3317-4617-a13e-d6a72278331e
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1daf02db-3317-4617-a13e-d6a72278331e",
      "name": "1daf02db-3317-4617-a13e-d6a72278331e",
      "status": "Failed",
      "startTime": "2026-10-22T22:42:59.591011904Z",
      "endTime": "2026-10-22T23:12:07.791955181Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.23.0-ec.1 is Partial (want Completed), started 20m52s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1daf02db-3317-4617-a13e-d6a72278331e
            correlation ID: 5bc687aa-4948-47f9-abde-ed1eae592853
            client request ID: 990e5092-5d1b-4b02-8e7c-312e0ee47aa5
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: HCP cluster rg-candidate-4-23-h5dw5xj4gsz5/cluster-candidate-4-23-q8dfp4 should provision
Unexpected error:
    <*fmt.wrapError | 0x202fbf30a20>: 
    failed to create HCP cluster cluster-candidate-4-23-q8dfp4: failed waiting for cluster="cluster-candidate-4-23-q8dfp4" in resourcegroup="rg-candidate-4-23-h5dw5xj4gsz5" to finish creating: GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1daf02db-3317-4617-a13e-d6a72278331e
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1daf02db-3317-4617-a13e-d6a72278331e",
      "name": "1daf02db-3317-4617-a13e-d6a72278331e",
      "status": "Failed",
      "startTime": "2026-10-22T22:42:59.591011904Z",
      "endTime": "2026-10-22T23:12:07.791955181Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.23.0-ec.1 is Partial (want Completed), started 20m52s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1daf02db-3317-4617-a13e-d6a72278331e
            correlation ID: 5bc687aa-4948-47f9-abde-ed1eae592853
            client request ID: 990e5092-5d1b-4b02-8e7c-312e0ee47aa5
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: HCP cluster rg-candidate-5-0-7v6qdnlhwvvb/cluster-candidate-5-0-9zpb54 should provision
Unexpected error:
    <*fmt.wrapError | 0x3a093dfe4480>: 
    failed to create HCP cluster cluster-candidate-5-0-9zpb54: failed waiting for cluster="cluster-candidate-5-0-9zpb54" in resourcegroup="rg-candidate-5-0-7v6qdnlhwvvb" to finish creating: GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23ba9cb2-0511-4188-a063-cf6d4e7f1fde
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23ba9cb2-0511-4188-a063-cf6d4e7f1fde",
      "name": "23ba9cb2-0511-4188-a063-cf6d4e7f1fde",
      "status": "Failed",
      "startTime": "2026-10-22T22:43:09.924601415Z",
      "endTime": "2026-10-22T23:12:18.105452223Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.7 is Partial (want Completed), started 21m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23ba9cb2-0511-4188-a063-cf6d4e7f1fde
            correlation ID: 1ba9603d-b743-4b6a-ad8e-362bed8e2691
            client request ID: 5ec430fe-78af-4487-8945-3607302f78b7
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: HCP cluster rg-candidate-5-0-7v6qdnlhwvvb/cluster-candidate-5-0-9zpb54 should provision
Unexpected error:
    <*fmt.wrapError | 0x3a093dfe4480>: 
    failed to create HCP cluster cluster-candidate-5-0-9zpb54: failed waiting for cluster="cluster-candidate-5-0-9zpb54" in resourcegroup="rg-candidate-5-0-7v6qdnlhwvvb" to finish creating: GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23ba9cb2-0511-4188-a063-cf6d4e7f1fde
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23ba9cb2-0511-4188-a063-cf6d4e7f1fde",
      "name": "23ba9cb2-0511-4188-a063-cf6d4e7f1fde",
      "status": "Failed",
      "startTime": "2026-10-22T22:43:09.924601415Z",
      "endTime": "2026-10-22T23:12:18.105452223Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.7 is Partial (want Completed), started 21m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3604736.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/23ba9cb2-0511-4188-a063-cf6d4e7f1fde
            correlation ID: 1ba9603d-b743-4b6a-ad8e-362bed8e2691
            client request ID: 5ec430fe-78af-4487-8945-3607302f78b7
occurred

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-22T22:57:13Z
  Ended:       2026-10-22T23:24:12Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3604736-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tk46t38vvtso9smm7df5vj1o9gclv0j-b7a9d8e5d1m5r3a", pod="router-b47fc45-hpmlk", region="canadacentral", severity="warning"
  Description: Pod ocm-arohcpci01-2tk46t38vvtso9smm7df5vj1o9gclv0j-b7a9d8e5d1m5r3a/router-b47fc45-hpmlk has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-22T22:58:13Z
  Ended:       2026-10-22T23:24:12Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j3604736-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tk46vmso1j9nnf1s1kifbbej15nl6s5-u2x7s9z5l4p4r7w", pod="router-7c878788f5-hkdfr", region="canadacentral", severity="warning"
  Description: Pod ocm-arohcpci01-2tk46vmso1j9nnf1s1kifbbej15nl6s5-u2x7s9z5l4p4r7w/router-7c878788f5-hkdfr has been in a non-ready state for longer than 5 minutes.

time=2026-10-22T22:17:12.835Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description="Step istio-upgrade\n  Kind: IstioUpgrade\n  AKSCluster: {{ svc.aks.name }}\n"
time=2026-10-22T22:17:14.824Z level=INFO msg="Istio upgrade -- cluster state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8144640-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false
time=2026-10-22T22:17:14.824Z level=INFO msg="Upgrading to svc.istio.versions target" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8144640-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35
time=2026-10-22T22:17:16.343Z level=INFO msg="Istio upgrade -- mesh state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8144640-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2
time=2026-10-22T22:17:16.343Z level=INFO msg="Starting canary -- installing target alongside current" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8144640-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-22T22:17:16.344Z level=INFO msg="Starting ARM operation" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j8144640-svc
time=2026-10-22T22:32:19.729Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err="failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded"

alert fired 21 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-22T22:23:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8144640-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-qm65f", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-qm65f has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-22T22:23:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8144640-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-646rk", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-646rk has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-22T22:23:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8144640-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-97mz4", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-97mz4 has been in a non-ready state for longer than 5 minutes.
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-22T22:23:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8144640-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-ztvz7", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-ztvz7 has been in a non-ready state for longer than 5 minutes.
Firing 5:
  State:       Fired (not resolved)
  Started:     2026-10-22T22:23:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8144640-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-6vncz", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-6vncz has been in a non-ready state for longer than 5 minutes.
Firing 6:
  State:       Fired (not resolved)
  Started:     2026-10-22T22:23:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8144640-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-vw4s5", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-vw4s5 has been in a non-ready state for longer than 5 minutes.
Firing 7:
  State:       Fired (not resolved)
  Started:     2026-10-22T22:23:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8144640-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-8tcjd", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-8tcjd has been in a non-ready state for longer than 5 minutes.
Firing 8:
  State:       Fired (not resolved)
  Started:     2026-10-22T22:23:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8144640-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-rf2c7", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-rf2c7 has been in a non-ready state for longer than 5 minutes.
Firing 9:
  State:       Fired (not resolved)
  Started:     2026-10-22T22:23:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8144640-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-tv8lq", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-tv8lq has been in a non-ready state for longer than 5 minutes.
Firing 10:
  State:       Fired (not resolved)
  Started:     2026-10-22T22:23:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8144640-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-74f8k", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-74f8k has been in a non-ready state for longer than 5 minutes.
Firing 11:
  State:       Fired (not resolved)
  Started:     2026-10-22T22:23:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8144640-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-x5msf", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-x5msf has been in a non-ready state for longer than 5 minutes.
Firing 12:
  State:       Fired (not resolved)
  Started:     2026-10-22T22:23:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8144640-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-hfcqc", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-hfcqc has been in a non-ready state for longer than 5 minutes.
Firing 13:
  State:       Fired (not resolved)
  Started:     2026-10-22T22:23:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8144640-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-nhww7", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-nhww7 has been in a non-ready state for longer than 5 minutes.
Firing 14:
  State:       Fired (not resolved)
  Started:     2026-10-22T22:23:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8144640-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-99qll", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-99qll has been in a non-ready state for longer than 5 minutes.
Firing 15:
  State:       Fired (not resolved)
  Started:     2026-10-22T22:23:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8144640-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-8zp7x", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-8zp7x has been in a non-ready state for longer than 5 minutes.
Firing 16:
  State:       Fired (not resolved)
  Started:     2026-10-22T22:23:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8144640-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-6lchz", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-6lchz has been in a non-ready state for longer than 5 minutes.
Firing 17:
  State:       Fired (not resolved)
  Started:     2026-10-22T22:23:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8144640-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-m29wl", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-m29wl has been in a non-ready state for longer than 5 minutes.
Firing 18:
  State:       Fired (not resolved)
  Started:     2026-10-22T22:23:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8144640-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-tvhb4", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-tvhb4 has been in a non-ready state for longer than 5 minutes.
Firing 19:
  State:       Fired (not resolved)
  Started:     2026-10-22T22:23:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8144640-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-t9zc6", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-t9zc6 has been in a non-ready state for longer than 5 minutes.
Firing 20:
  State:       Fired (not resolved)
  Started:     2026-10-22T22:23:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8144640-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-47967", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-47967 has been in a non-ready state for longer than 5 minutes.
Firing 21:
  State:       Fired (not resolved)
  Started:     2026-10-22T22:23:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j8144640-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-2bsv7", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-2bsv7 has been in a non-ready state for longer than 5 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jppjcpm2vn5h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x32b05a79ea40>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jppjcpm2vn5h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: dev/canadacentral/ci01-j8160256-mgmt-2/ocm-arohcpci01-2tk43am3a21h27v18e3q17o5d9maalpn-l9n3u6m4v9a3e5a: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jppjcpm2vn5h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: dev/canadacentral/ci01-j8160256-mgmt-2/ocm-arohcpci01-2tk43am3a21h27v18e3q17o5d9maalpn-l9n3u6m4v9a3e5a: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        err: <*errors.errorString | 0x32b05c5389f0>{
            s: "dev/canadacentral/ci01-j8160256-mgmt-2/ocm-arohcpci01-2tk43am3a21h27v18e3q17o5d9maalpn-l9n3u6m4v9a3e5a: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jppjcpm2vn5h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x32b05a79ea40>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jppjcpm2vn5h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: dev/canadacentral/ci01-j8160256-mgmt-2/ocm-arohcpci01-2tk43am3a21h27v18e3q17o5d9maalpn-l9n3u6m4v9a3e5a: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jppjcpm2vn5h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: dev/canadacentral/ci01-j8160256-mgmt-2/ocm-arohcpci01-2tk43am3a21h27v18e3q17o5d9maalpn-l9n3u6m4v9a3e5a: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        err: <*errors.errorString | 0x32b05c5389f0>{
            s: "dev/canadacentral/ci01-j8160256-mgmt-2/ocm-arohcpci01-2tk43am3a21h27v18e3q17o5d9maalpn-l9n3u6m4v9a3e5a: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        },
    }
occurred

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-22T22:34:51Z
  Ended:       2026-10-22T22:41:02Z
  Severity:    Sev3
  Description: Cosmos DB normalized RU consumption is above 95% averaged over a 5-minute window, evaluated every minute. This catches short RU-saturation bursts (e.g. mass cluster creation at the start of E2E tests) that exceed provisioned throughput and can lead to 429 throttling before the 10-minute sustained alert fires. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-22T22:23:37Z
  Ended:       2026-10-22T22:51:36Z
  Severity:    Sev3
  Description: Cosmos DB normalized RU consumption is above 70% averaged over a 15-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:193]: failed to verify simple web app runs on cluster "cluster-candidate-4-22-8s2f2l"
Unexpected error:
    <*fmt.wrapError | 0x30cafd6d0a20>: 
    DNS for route host agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded",
        err: <*fmt.wrapError | 0x30cafd44e600>{
            msg: "DNS for agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:193]: failed to verify simple web app runs on cluster "cluster-candidate-4-22-8s2f2l"
Unexpected error:
    <*fmt.wrapError | 0x30cafd6d0a20>: 
    DNS for route host agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded",
        err: <*fmt.wrapError | 0x30cafd44e600>{
            msg: "DNS for agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-56lgx.apps.aro.y9c6b2i7y5a3u5n.l4il.j7335040.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred

time=2026-10-22T21:15:19.830Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 description="Step swift-vnet\n  Kind: Shell\n  Command: ./swift-vnet.sh\n"
time=2026-10-22T21:15:40.577Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 err="stamp 1: error running Shell Step, failed to execute shell command: Launching container group swift-vnet-aks-net as globalMSI...\nWaiting for swift-vnet-aks-net to finish (state: pending, elapsed 2s)...\n✗ Container group swift-vnet-aks-net failed to provision\n\n exit status 1"

time=2026-10-22T21:04:17.903Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description="Step cluster\n  Kind: ARM\n  Template: templates/mgmt-cluster.bicep\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam"
time=2026-10-22T21:04:18.672Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1
time=2026-10-22T21:04:19.255Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=6a5336e24f93f554fd8b0d65c59b856837a40a2fdae6e7d0fbc40c8888d8d17e portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j3800704-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6a5336e24f93f554fd8b0d65c59b856837a40a2fdae6e7d0fbc40c8888d8d17e
time=2026-10-22T21:07:49.884Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err="stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3800704-mgmt-1/providers/Microsoft.Resources/deployments/6a5336e24f93f554fd8b0d65c59b856837a40a2fdae6e7d0fbc40c8888d8d17e/operationStatuses/08584089038265307890\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"DeploymentActive\\\",\\r\\n    \\\"message\\\": \\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3800704-mgmt-1/providers/Microsoft.Resources/deployments/fe73d360-7d74-562a-b1ee-44865af0a11a' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '10/22/2026 9:06:20 PM' with correlationId 'fe6d7e33-2867-4647-8fe8-d9c4bbb9af94', and will expire at '10/29/2026 9:06:20 PM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\"\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-22T21:19:53Z
  Severity:    Sev2
  Labels:      alertname="PrometheusJobUp", cluster="ci01-j3800704-mgmt-1", component="monitoring-infrastructure", severity="critical"
  Description: Prometheus has not been reachable for the past 10 minutes.
This may indicate that the Prometheus server is down, unreachable due to network issues, or experiencing a crash loop.
Check the status of the Prometheus pods, service endpoints, and network connectivity.

alert fired 1 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-22T21:30:04Z
  Severity:    Sev3
  Labels:      alertname="ArobitForwarderJobUp", cluster="ci01-j3800704-mgmt-1", component="monitoring-infrastructure", severity="warning"
  Description: The Arobit forwarder metrics endpoint on cluster ci01-j3800704-mgmt-1 has been unreachable for 15 minutes.

alert fired 2 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-22T21:15:12Z
  Severity:    Sev3
  Labels:      alertname="MaestroAgentMetricsProxyDown", cluster="ci01-j3800704-mgmt-2", component="monitoring-infrastructure", region="canadacentral", severity="3"
  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-22T21:24:13Z
  Severity:    Sev3
  Labels:      alertname="MaestroAgentMetricsProxyDown", cluster="ci01-j3800704-mgmt-1", component="monitoring-infrastructure", severity="3"
  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0xc8f6d4d40a8>: 
    resource group "rg-candidate-4-23-wv4m7dxwvslv": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cluster-candidate-4-23-llx74k" in resourcegroup="rg-candidate-4-23-wv4m7dxwvslv" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38d6d256-b8f5-43d6-9bea-bc7db4e81a10
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38d6d256-b8f5-43d6-9bea-bc7db4e81a10",
      "name": "38d6d256-b8f5-43d6-9bea-bc7db4e81a10",
      "status": "Failed",
      "startTime": "2026-10-22T19:58:57.002004594Z",
      "endTime": "2026-10-22T20:23:04.238386188Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38d6d256-b8f5-43d6-9bea-bc7db4e81a10
            correlation ID: 89679606-4473-4b98-af12-ab459a5b124c
            client request ID: e90666ef-7f96-44ec-8272-834002e9a78d
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0xc8f6d4d40a8>: 
    resource group "rg-candidate-4-23-wv4m7dxwvslv": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cluster-candidate-4-23-llx74k" in resourcegroup="rg-candidate-4-23-wv4m7dxwvslv" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38d6d256-b8f5-43d6-9bea-bc7db4e81a10
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38d6d256-b8f5-43d6-9bea-bc7db4e81a10",
      "name": "38d6d256-b8f5-43d6-9bea-bc7db4e81a10",
      "status": "Failed",
      "startTime": "2026-10-22T19:58:57.002004594Z",
      "endTime": "2026-10-22T20:23:04.238386188Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38d6d256-b8f5-43d6-9bea-bc7db4e81a10
            correlation ID: 89679606-4473-4b98-af12-ab459a5b124c
            client request ID: e90666ef-7f96-44ec-8272-834002e9a78d
occurred

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x3e4a7eaec120>: 
    resource group "kusto-logs-bjqbcmj9fh99": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="engineering-hcp-cluster" in resourcegroup="kusto-logs-bjqbcmj9fh99" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/133a0c79-b758-4022-b9d3-abe3437fdb7e
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/133a0c79-b758-4022-b9d3-abe3437fdb7e",
      "name": "133a0c79-b758-4022-b9d3-abe3437fdb7e",
      "status": "Failed",
      "startTime": "2026-10-22T20:01:21.775547749Z",
      "endTime": "2026-10-22T20:25:24.154021571Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/133a0c79-b758-4022-b9d3-abe3437fdb7e
            correlation ID: 876f3f40-3d0a-4f9d-b0a1-13067115d7f2
            client request ID: a6519fef-aba9-4692-8b59-2cc44c90a17d
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x3e4a7eaec120>: 
    resource group "kusto-logs-bjqbcmj9fh99": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="engineering-hcp-cluster" in resourcegroup="kusto-logs-bjqbcmj9fh99" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/133a0c79-b758-4022-b9d3-abe3437fdb7e
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/133a0c79-b758-4022-b9d3-abe3437fdb7e",
      "name": "133a0c79-b758-4022-b9d3-abe3437fdb7e",
      "status": "Failed",
      "startTime": "2026-10-22T20:01:21.775547749Z",
      "endTime": "2026-10-22T20:25:24.154021571Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/133a0c79-b758-4022-b9d3-abe3437fdb7e
            correlation ID: 876f3f40-3d0a-4f9d-b0a1-13067115d7f2
            client request ID: a6519fef-aba9-4692-8b59-2cc44c90a17d
occurred

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x5f7edbba210>: 
    resource group "feature-aggregation-9bffc7pfd9cf": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="agg-cluster" in resourcegroup="feature-aggregation-9bffc7pfd9cf" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80845250-6487-4bd8-bc2d-fcba49147d69
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80845250-6487-4bd8-bc2d-fcba49147d69",
      "name": "80845250-6487-4bd8-bc2d-fcba49147d69",
      "status": "Failed",
      "startTime": "2026-10-22T20:00:56.002177295Z",
      "endTime": "2026-10-22T20:25:04.157804188Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80845250-6487-4bd8-bc2d-fcba49147d69
            correlation ID: b47a1ca2-3039-4616-a8a8-424022677398
            client request ID: da4cb843-dd82-426b-ae29-8754ca365c47
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x5f7edbba210>: 
    resource group "feature-aggregation-9bffc7pfd9cf": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="agg-cluster" in resourcegroup="feature-aggregation-9bffc7pfd9cf" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80845250-6487-4bd8-bc2d-fcba49147d69
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80845250-6487-4bd8-bc2d-fcba49147d69",
      "name": "80845250-6487-4bd8-bc2d-fcba49147d69",
      "status": "Failed",
      "startTime": "2026-10-22T20:00:56.002177295Z",
      "endTime": "2026-10-22T20:25:04.157804188Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/80845250-6487-4bd8-bc2d-fcba49147d69
            correlation ID: b47a1ca2-3039-4616-a8a8-424022677398
            client request ID: da4cb843-dd82-426b-ae29-8754ca365c47
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-kb7qmr with version 4.21.39
Unexpected error:
    <*fmt.wrapError | 0x28069869e800>: 
    failed to create HCP cluster np-version-upgrade-cluster-kb7qmr: failed waiting for cluster="np-version-upgrade-cluster-kb7qmr" in resourcegroup="rg-np-version-upgrade-kb7qmr-c47gf99np5dv" to finish creating: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fea1990-392a-4c48-ad2a-e3b2aaa2a5d0
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fea1990-392a-4c48-ad2a-e3b2aaa2a5d0",
      "name": "9fea1990-392a-4c48-ad2a-e3b2aaa2a5d0",
      "status": "Failed",
      "startTime": "2026-10-22T19:49:25.431008286Z",
      "endTime": "2026-10-22T20:18:34.116415534Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fea1990-392a-4c48-ad2a-e3b2aaa2a5d0
            correlation ID: 0f045ea6-0c55-4d5b-bd0c-acde888327dd
            client request ID: cee3d7d5-1361-4cfc-b548-f2d63668a2b7
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-kb7qmr with version 4.21.39
Unexpected error:
    <*fmt.wrapError | 0x28069869e800>: 
    failed to create HCP cluster np-version-upgrade-cluster-kb7qmr: failed waiting for cluster="np-version-upgrade-cluster-kb7qmr" in resourcegroup="rg-np-version-upgrade-kb7qmr-c47gf99np5dv" to finish creating: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fea1990-392a-4c48-ad2a-e3b2aaa2a5d0
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fea1990-392a-4c48-ad2a-e3b2aaa2a5d0",
      "name": "9fea1990-392a-4c48-ad2a-e3b2aaa2a5d0",
      "status": "Failed",
      "startTime": "2026-10-22T19:49:25.431008286Z",
      "endTime": "2026-10-22T20:18:34.116415534Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fea1990-392a-4c48-ad2a-e3b2aaa2a5d0
            correlation ID: 0f045ea6-0c55-4d5b-bd0c-acde888327dd
            client request ID: cee3d7d5-1361-4cfc-b548-f2d63668a2b7
occurred

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x1445b90de228>: 
    resource group "rg-candidate-4-21-td4mhfs4j8v5": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cluster-candidate-4-21-grnnwp" in resourcegroup="rg-candidate-4-21-td4mhfs4j8v5" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cbbdf31a-b04f-4e26-9d48-9d89c5d12d8e
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cbbdf31a-b04f-4e26-9d48-9d89c5d12d8e",
      "name": "cbbdf31a-b04f-4e26-9d48-9d89c5d12d8e",
      "status": "Failed",
      "startTime": "2026-10-22T19:56:06.651640707Z",
      "endTime": "2026-10-22T20:20:14.650326027Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cbbdf31a-b04f-4e26-9d48-9d89c5d12d8e
            correlation ID: ecab9896-2020-4354-a4df-380a8a550a70
            client request ID: 3bfbbe06-8120-4d72-8516-3e9a6561ddcd
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x1445b90de228>: 
    resource group "rg-candidate-4-21-td4mhfs4j8v5": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cluster-candidate-4-21-grnnwp" in resourcegroup="rg-candidate-4-21-td4mhfs4j8v5" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cbbdf31a-b04f-4e26-9d48-9d89c5d12d8e
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cbbdf31a-b04f-4e26-9d48-9d89c5d12d8e",
      "name": "cbbdf31a-b04f-4e26-9d48-9d89c5d12d8e",
      "status": "Failed",
      "startTime": "2026-10-22T19:56:06.651640707Z",
      "endTime": "2026-10-22T20:20:14.650326027Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cbbdf31a-b04f-4e26-9d48-9d89c5d12d8e
            correlation ID: ecab9896-2020-4354-a4df-380a8a550a70
            client request ID: 3bfbbe06-8120-4d72-8516-3e9a6561ddcd
occurred

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x1e7a4ac381c8>: 
    resource group "rg-candidate-5-0-54ll7ff5h97p": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cluster-candidate-5-0-bdl7b6" in resourcegroup="rg-candidate-5-0-54ll7ff5h97p" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56d97582-5397-4afa-ae32-c0de69fc1588
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56d97582-5397-4afa-ae32-c0de69fc1588",
      "name": "56d97582-5397-4afa-ae32-c0de69fc1588",
      "status": "Failed",
      "startTime": "2026-10-22T20:00:21.519712195Z",
      "endTime": "2026-10-22T20:24:24.151438373Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56d97582-5397-4afa-ae32-c0de69fc1588
            correlation ID: 086aa11a-7ea6-463b-a979-85ac6ded58d4
            client request ID: 7e8f297b-44a9-439d-b3c8-54c4cf3d8847
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x1e7a4ac381c8>: 
    resource group "rg-candidate-5-0-54ll7ff5h97p": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cluster-candidate-5-0-bdl7b6" in resourcegroup="rg-candidate-5-0-54ll7ff5h97p" to finish deleting: GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56d97582-5397-4afa-ae32-c0de69fc1588
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56d97582-5397-4afa-ae32-c0de69fc1588",
      "name": "56d97582-5397-4afa-ae32-c0de69fc1588",
      "status": "Failed",
      "startTime": "2026-10-22T20:00:21.519712195Z",
      "endTime": "2026-10-22T20:24:24.151438373Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7389824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/56d97582-5397-4afa-ae32-c0de69fc1588
            correlation ID: 086aa11a-7ea6-463b-a979-85ac6ded58d4
            client request ID: 7e8f297b-44a9-439d-b3c8-54c4cf3d8847
occurred

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0xb52cc494270>: 
    resource group "rg-candidate-4-22-dbz2f2dgxvsz": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cluster-candidate-4-22-5sf4vm" in resourcegroup="rg-candidate-4-22-dbz2f2dgxvsz" to finish deleting: GET https://rp.j0711680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e97180c3-ec2d-41de-ba05-4c97d3788b0c
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e97180c3-ec2d-41de-ba05-4c97d3788b0c",
      "name": "e97180c3-ec2d-41de-ba05-4c97d3788b0c",
      "status": "Failed",
      "startTime": "2026-10-22T19:51:01.626448313Z",
      "endTime": "2026-10-22T20:15:04.909018235Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0711680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e97180c3-ec2d-41de-ba05-4c97d3788b0c
            correlation ID: 06dd90eb-d763-414b-9dbc-6cbbd01f522d
            client request ID: 134d05fc-4c49-4a17-ae46-0db6a5b33c79
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0xb52cc494270>: 
    resource group "rg-candidate-4-22-dbz2f2dgxvsz": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cluster-candidate-4-22-5sf4vm" in resourcegroup="rg-candidate-4-22-dbz2f2dgxvsz" to finish deleting: GET https://rp.j0711680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e97180c3-ec2d-41de-ba05-4c97d3788b0c
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e97180c3-ec2d-41de-ba05-4c97d3788b0c",
      "name": "e97180c3-ec2d-41de-ba05-4c97d3788b0c",
      "status": "Failed",
      "startTime": "2026-10-22T19:51:01.626448313Z",
      "endTime": "2026-10-22T20:15:04.909018235Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0711680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e97180c3-ec2d-41de-ba05-4c97d3788b0c
            correlation ID: 06dd90eb-d763-414b-9dbc-6cbbd01f522d
            client request ID: 134d05fc-4c49-4a17-ae46-0db6a5b33c79
occurred

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x20e75a338120>: 
    resource group "rg-candidate-4-23-wm8dczw7qltp": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cluster-candidate-4-23-q6d2hl" in resourcegroup="rg-candidate-4-23-wm8dczw7qltp" to finish deleting: GET https://rp.j0711680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7182a919-4fbb-49ce-83aa-dd6eb45ba683
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7182a919-4fbb-49ce-83aa-dd6eb45ba683",
      "name": "7182a919-4fbb-49ce-83aa-dd6eb45ba683",
      "status": "Failed",
      "startTime": "2026-10-22T19:49:32.282155892Z",
      "endTime": "2026-10-22T20:13:36.499422253Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0711680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7182a919-4fbb-49ce-83aa-dd6eb45ba683
            correlation ID: ccfc9272-4802-4cdf-a65e-9aaeb8062ade
            client request ID: e0488b7a-701b-4302-8c93-fb4506094516
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x20e75a338120>: 
    resource group "rg-candidate-4-23-wm8dczw7qltp": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cluster-candidate-4-23-q6d2hl" in resourcegroup="rg-candidate-4-23-wm8dczw7qltp" to finish deleting: GET https://rp.j0711680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7182a919-4fbb-49ce-83aa-dd6eb45ba683
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7182a919-4fbb-49ce-83aa-dd6eb45ba683",
      "name": "7182a919-4fbb-49ce-83aa-dd6eb45ba683",
      "status": "Failed",
      "startTime": "2026-10-22T19:49:32.282155892Z",
      "endTime": "2026-10-22T20:13:36.499422253Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0711680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7182a919-4fbb-49ce-83aa-dd6eb45ba683
            correlation ID: ccfc9272-4802-4cdf-a65e-9aaeb8062ade
            client request ID: e0488b7a-701b-4302-8c93-fb4506094516
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.
Expected success, but got an error:
    <*errors.errorString | 0x3863a797e030>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1371, hypershift=10239, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1371, hypershift=10239, acm-agent=0",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.
Expected success, but got an error:
    <*errors.errorString | 0x3863a797e030>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1371, hypershift=10239, acm-agent=0
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1371, hypershift=10239, acm-agent=0",
    }

alert fired 4 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-22T19:39:05Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j0711680-mgmt-2", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.242:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0711680/providers/microsoft.monitor/accounts/services-j0711680", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7c96c6b89d-86mcm", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="canadacentral", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="44790cfe-d60d-4b27-a39b-93e0f60af9be"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-86mcm (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-22T19:40:06Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j0711680-mgmt-1", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.234:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0711680/providers/microsoft.monitor/accounts/services-j0711680", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7c96c6b89d-8wz92", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="canadacentral", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="d334bde9-8444-442b-a710-d97c48525e23"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-8wz92 (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-22T19:41:10Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j0711680-mgmt-2", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.242:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0711680/providers/microsoft.monitor/accounts/services-j0711680", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7c96c6b89d-86mcm", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="canadacentral", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="44790cfe-d60d-4b27-a39b-93e0f60af9be"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-86mcm (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-22T19:42:08Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j0711680-mgmt-1", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.234:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0711680/providers/microsoft.monitor/accounts/services-j0711680", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7c96c6b89d-8wz92", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="canadacentral", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="d334bde9-8444-442b-a710-d97c48525e23"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-8wz92 (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").

time=2026-10-22T18:49:27.700Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=2 description="Step swift-vnet\n  Kind: Shell\n  Command: ./swift-vnet.sh\n"
time=2026-10-22T18:49:46.822Z level=ERROR msg="failed to remove step Azure CLI config directory" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=2 err="unlinkat /tmp/azure-cli-step-config-1336382318: directory not empty" dir=/tmp/azure-cli-step-config-1336382318
time=2026-10-22T18:49:46.822Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=2 err="stamp 2: error running Shell Step, failed to execute shell command: Launching container group swift-vnet-aks-net as globalMSI...\nWaiting for swift-vnet-aks-net to finish (state: pending, elapsed 1s)...\n✗ Container group swift-vnet-aks-net failed to provision\n\n exit status 1"

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools
Expected success, but got an error:
    <*fmt.wrapError | 0x18d647b87940>: 
    failed to create NodePool np-deletable: failed waiting for nodepool="np-deletable" for cluster "np-delete-hcp-cluster" in resourcegroup="nodepool-delete-8q2cclgqxhlk" to finish creating: GET https://rp.j9423104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0d5e7b9b-6357-498c-88f8-d878e3a54b53
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0d5e7b9b-6357-498c-88f8-d878e3a54b53",
      "name": "0d5e7b9b-6357-498c-88f8-d878e3a54b53",
      "status": "Failed",
      "startTime": "2026-10-22T18:25:27.469650053Z",
      "endTime": "2026-10-22T18:44:36.112402686Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine q7p8r2d0k2d6y0r-np-deletable-dg4tx-qm5s4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8q2cclgqxhlk--managed/providers/Microsoft.Compute/virtualMachines/q7p8r2d0k2d6y0r-np-deletable-dg4tx-qm5s4 to exist\nMachine q7p8r2d0k2d6y0r-np-deletable-dg4tx-wdnnf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8q2cclgqxhlk--managed/providers/Microsoft.Compute/virtualMachines/q7p8r2d0k2d6y0r-np-deletable-dg4tx-wdnnf to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine q7p8r2d0k2d6y0r-np-deletable-dg4tx-qm5s4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8q2cclgqxhlk--managed/providers/Microsoft.Compute/virtualMachines/q7p8r2d0k2d6y0r-np-deletable-dg4tx-qm5s4 to exist\nMachine q7p8r2d0k2d6y0r-np-deletable-dg4tx-wdnnf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8q2cclgqxhlk--managed/providers/Microsoft.Compute/virtualMachines/q7p8r2d0k2d6y0r-np-deletable-dg4tx-wdnnf to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9423104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0d5e7b9b-6357-498c-88f8-d878e3a54b53
            correlation ID: de487052-0457-4f7a-86bf-23162f549db5
            client request ID: 8a9f42ff-db11-48e8-af3f-fae5e92dcff9
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools
Expected success, but got an error:
    <*fmt.wrapError | 0x18d647b87940>: 
    failed to create NodePool np-deletable: failed waiting for nodepool="np-deletable" for cluster "np-delete-hcp-cluster" in resourcegroup="nodepool-delete-8q2cclgqxhlk" to finish creating: GET https://rp.j9423104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0d5e7b9b-6357-498c-88f8-d878e3a54b53
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0d5e7b9b-6357-498c-88f8-d878e3a54b53",
      "name": "0d5e7b9b-6357-498c-88f8-d878e3a54b53",
      "status": "Failed",
      "startTime": "2026-10-22T18:25:27.469650053Z",
      "endTime": "2026-10-22T18:44:36.112402686Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine q7p8r2d0k2d6y0r-np-deletable-dg4tx-qm5s4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8q2cclgqxhlk--managed/providers/Microsoft.Compute/virtualMachines/q7p8r2d0k2d6y0r-np-deletable-dg4tx-qm5s4 to exist\nMachine q7p8r2d0k2d6y0r-np-deletable-dg4tx-wdnnf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8q2cclgqxhlk--managed/providers/Microsoft.Compute/virtualMachines/q7p8r2d0k2d6y0r-np-deletable-dg4tx-wdnnf to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine q7p8r2d0k2d6y0r-np-deletable-dg4tx-qm5s4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8q2cclgqxhlk--managed/providers/Microsoft.Compute/virtualMachines/q7p8r2d0k2d6y0r-np-deletable-dg4tx-qm5s4 to exist\nMachine q7p8r2d0k2d6y0r-np-deletable-dg4tx-wdnnf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-8q2cclgqxhlk--managed/providers/Microsoft.Compute/virtualMachines/q7p8r2d0k2d6y0r-np-deletable-dg4tx-wdnnf to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9423104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0d5e7b9b-6357-498c-88f8-d878e3a54b53
            correlation ID: de487052-0457-4f7a-86bf-23162f549db5
            client request ID: 8a9f42ff-db11-48e8-af3f-fae5e92dcff9

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43
Unexpected error:
    <*fmt.wrapError | 0x12f37f64c920>: 
    failed to create NodePool npupgrade-4-20: failed waiting for nodepool="npupgrade-4-20" for cluster "np-version-upgrade-cluster-7h867b" in resourcegroup="rg-np-version-upgrade-7h867b-ccdxwvgmcw5k" to finish creating: GET https://rp.j7712512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/800b48a8-8e04-4c8c-a76f-848043b4ca68
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/800b48a8-8e04-4c8c-a76f-848043b4ca68",
      "name": "800b48a8-8e04-4c8c-a76f-848043b4ca68",
      "status": "Failed",
      "startTime": "2026-10-22T18:35:17.000023517Z",
      "endTime": "2026-10-22T18:54:25.220281009Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine m6p8x9g7y9z8s8d-npupgrade-4-20-rshvh-svgmr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7h867b-ccdxwvgmcw5k-np-upgrade-7h-d37091ba/providers/Microsoft.Compute/virtualMachines/m6p8x9g7y9z8s8d-npupgrade-4-20-rshvh-svgmr to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine m6p8x9g7y9z8s8d-npupgrade-4-20-rshvh-svgmr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7h867b-ccdxwvgmcw5k-np-upgrade-7h-d37091ba/providers/Microsoft.Compute/virtualMachines/m6p8x9g7y9z8s8d-npupgrade-4-20-rshvh-svgmr to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7712512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/800b48a8-8e04-4c8c-a76f-848043b4ca68
            correlation ID: 4baff6fc-a012-46cd-a498-5faf00b70f7d
            client request ID: 6aa50b83-5102-4e98-a262-a0727625bcc5
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43
Unexpected error:
    <*fmt.wrapError | 0x12f37f64c920>: 
    failed to create NodePool npupgrade-4-20: failed waiting for nodepool="npupgrade-4-20" for cluster "np-version-upgrade-cluster-7h867b" in resourcegroup="rg-np-version-upgrade-7h867b-ccdxwvgmcw5k" to finish creating: GET https://rp.j7712512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/800b48a8-8e04-4c8c-a76f-848043b4ca68
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/800b48a8-8e04-4c8c-a76f-848043b4ca68",
      "name": "800b48a8-8e04-4c8c-a76f-848043b4ca68",
      "status": "Failed",
      "startTime": "2026-10-22T18:35:17.000023517Z",
      "endTime": "2026-10-22T18:54:25.220281009Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine m6p8x9g7y9z8s8d-npupgrade-4-20-rshvh-svgmr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7h867b-ccdxwvgmcw5k-np-upgrade-7h-d37091ba/providers/Microsoft.Compute/virtualMachines/m6p8x9g7y9z8s8d-npupgrade-4-20-rshvh-svgmr to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine m6p8x9g7y9z8s8d-npupgrade-4-20-rshvh-svgmr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7h867b-ccdxwvgmcw5k-np-upgrade-7h-d37091ba/providers/Microsoft.Compute/virtualMachines/m6p8x9g7y9z8s8d-npupgrade-4-20-rshvh-svgmr to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7712512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/800b48a8-8e04-4c8c-a76f-848043b4ca68
            correlation ID: 4baff6fc-a012-46cd-a498-5faf00b70f7d
            client request ID: 6aa50b83-5102-4e98-a262-a0727625bcc5
occurred

time=2026-10-22T17:11:01.934Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 description="Step deploy\n Kind: Helm\n"
time=2026-10-22T17:11:02.617Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 values="map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:72abeb657cfbdd6a5891fc102a16429875d70c5932ad5994524d0246db914013 imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:55a0ad895a8f10193bb070883194449f5252c057353887af1ba0767760a0d2f0 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:<nil>]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]"
time=2026-10-22T17:11:02.617Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2
time=2026-10-22T17:11:03.344Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 namespace=secret-sync-controller
time=2026-10-22T17:11:03.700Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 dryRun=false
time=2026-10-22T17:11:03.700Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2
time=2026-10-22T17:11:03.826Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2
time=2026-10-22T17:16:10.104Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err="resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\ncontext deadline exceeded"
time=2026-10-22T17:16:10.104Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err="stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\ncontext deadline exceeded"

alert fired 12 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-22T17:16:34Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2047104-mgmt-2", component="kubernetes-infrastructure", namespace="multicluster-engine", pod="multicluster-engine-operator-79c768d748-gtmch", region="westus3", severity="warning"
  Description: Pod multicluster-engine/multicluster-engine-operator-79c768d748-gtmch has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-22T17:17:34Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2047104-mgmt-2", component="kubernetes-infrastructure", namespace="mgmt-agent", pod="mgmt-agent-58787656fc-vpngf", region="westus3", severity="warning"
  Description: Pod mgmt-agent/mgmt-agent-58787656fc-vpngf has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-22T17:17:34Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2047104-mgmt-2", component="kubernetes-infrastructure", namespace="kube-applier", pod="kube-applier-6d4c66b65c-cjgt4", region="westus3", severity="warning"
  Description: Pod kube-applier/kube-applier-6d4c66b65c-cjgt4 has been in a non-ready state for longer than 5 minutes.
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-22T17:17:34Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2047104-mgmt-2", component="kubernetes-infrastructure", namespace="secret-sync-controller", pod="secrets-store-sync-controller-manager-7f4b9bc5df-pcxv2", region="westus3", severity="warning"
  Description: Pod secret-sync-controller/secrets-store-sync-controller-manager-7f4b9bc5df-pcxv2 has been in a non-ready state for longer than 5 minutes.
Firing 5:
  State:       Fired (not resolved)
  Started:     2026-10-22T17:17:34Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2047104-mgmt-2", component="kubernetes-infrastructure", namespace="hypershift-sharedingress", pod="router-65664496f6-j6f62", region="westus3", severity="warning"
  Description: Pod hypershift-sharedingress/router-65664496f6-j6f62 has been in a non-ready state for longer than 5 minutes.
Firing 6:
  State:       Fired (not resolved)
  Started:     2026-10-22T17:18:35Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2047104-mgmt-2", component="kubernetes-infrastructure", namespace="multicluster-engine", pod="ocm-webhook-68676899fd-plgpk", region="westus3", severity="warning"
  Description: Pod multicluster-engine/ocm-webhook-68676899fd-plgpk has been in a non-ready state for longer than 5 minutes.
Firing 7:
  State:       Fired (not resolved)
  Started:     2026-10-22T17:18:35Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2047104-mgmt-2", component="kubernetes-infrastructure", namespace="multicluster-engine", pod="managedcluster-import-controller-v2-6cb46794f7-kfk7p", region="westus3", severity="warning"
  Description: Pod multicluster-engine/managedcluster-import-controller-v2-6cb46794f7-kfk7p has been in a non-ready state for longer than 5 minutes.
Firing 8:
  State:       Fired (not resolved)
  Started:     2026-10-22T17:18:35Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2047104-mgmt-2", component="kubernetes-infrastructure", namespace="open-cluster-management-agent-addon", pod="klusterlet-addon-workmgr-58f4f6fb5b-8mhhk", region="westus3", severity="warning"
  Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-58f4f6fb5b-8mhhk has been in a non-ready state for longer than 5 minutes.
Firing 9:
  State:       Fired (not resolved)
  Started:     2026-10-22T17:18:35Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2047104-mgmt-2", component="kubernetes-infrastructure", namespace="multicluster-engine", pod="ocm-controller-859484fc49-6spbd", region="westus3", severity="warning"
  Description: Pod multicluster-engine/ocm-controller-859484fc49-6spbd has been in a non-ready state for longer than 5 minutes.
Firing 10:
  State:       Fired (not resolved)
  Started:     2026-10-22T17:18:35Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2047104-mgmt-2", component="kubernetes-infrastructure", namespace="multicluster-engine", pod="cluster-manager-c5d6c587f-cx7xv", region="westus3", severity="warning"
  Description: Pod multicluster-engine/cluster-manager-c5d6c587f-cx7xv has been in a non-ready state for longer than 5 minutes.
Firing 11:
  State:       Fired (not resolved)
  Started:     2026-10-22T17:19:35Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2047104-mgmt-2", component="kubernetes-infrastructure", namespace="open-cluster-management-agent", pod="klusterlet-agent-54d5df574-f7jps", region="westus3", severity="warning"
  Description: Pod open-cluster-management-agent/klusterlet-agent-54d5df574-f7jps has been in a non-ready state for longer than 5 minutes.
Firing 12:
  State:       Fired (not resolved)
  Started:     2026-10-22T17:19:35Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2047104-mgmt-2", component="kubernetes-infrastructure", namespace="open-cluster-management-agent", pod="klusterlet-9574f7d67-8mdfx", region="westus3", severity="warning"
  Description: Pod open-cluster-management-agent/klusterlet-9574f7d67-8mdfx has been in a non-ready state for longer than 5 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x20c3ca48efe0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j7289984-mgmt-2/ocm-arohcpci01-2tjvfsmjml5p3r8raim1iagd9p4j2g3h-z7y9i1y5r7p5s7j: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j7289984-mgmt-2/ocm-arohcpci01-2tjvfsmjml5p3r8raim1iagd9p4j2g3h-z7y9i1y5r7p5s7j: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1",
        err: <*verifiers.pollObservationError | 0x20c3ca48ef80>{
            observation: <*errors.errorString | 0x20c3cb6d0a90>{
                s: "dev/centralus/ci01-j7289984-mgmt-2/ocm-arohcpci01-2tjvfsmjml5p3r8raim1iagd9p4j2g3h-z7y9i1y5r7p5s7j: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1",
            },
            interruption: <*fmt.wrapError | 0x20c3ca48ef60>{
                msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-22T17:19:49.1663237Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                err: <*errors.Error | 0x20c3c94b2370>{
                    Op: 1,
                    Kind: 7,
                    Err: <*fmt.wrapError | 0x20c3ca48ef40>{
                        msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-22T17:19:49.1663237Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                        err: <*url.Error | 0x20c3cb422450>{
                            Op: "Post",
                            URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                            Err: <context.deadlineExceededError>{},
                        },
                    },
                    restErrMsg: nil,
                    decoded: nil,
                    permanent: false,
                    inner: nil,
                },
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x20c3ca48efe0>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j7289984-mgmt-2/ocm-arohcpci01-2tjvfsmjml5p3r8raim1iagd9p4j2g3h-z7y9i1y5r7p5s7j: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j7289984-mgmt-2/ocm-arohcpci01-2tjvfsmjml5p3r8raim1iagd9p4j2g3h-z7y9i1y5r7p5s7j: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1",
        err: <*verifiers.pollObservationError | 0x20c3ca48ef80>{
            observation: <*errors.errorString | 0x20c3cb6d0a90>{
                s: "dev/centralus/ci01-j7289984-mgmt-2/ocm-arohcpci01-2tjvfsmjml5p3r8raim1iagd9p4j2g3h-z7y9i1y5r7p5s7j: expected all current routers >= desired replicas > 0; desired=3 current=1 rows=1",
            },
            interruption: <*fmt.wrapError | 0x20c3ca48ef60>{
                msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-22T17:19:49.1663237Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                err: <*errors.Error | 0x20c3c94b2370>{
                    Op: 1,
                    Kind: 7,
                    Err: <*fmt.wrapError | 0x20c3ca48ef40>{
                        msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-8nllkjhjqw54/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-22T17:19:49.1663237Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                        err: <*url.Error | 0x20c3cb422450>{
                            Op: "Post",
                            URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                            Err: <context.deadlineExceededError>{},
                        },
                    },
                    restErrMsg: nil,
                    decoded: nil,
                    permanent: false,
                    inner: nil,
                },
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret
Unexpected error:
    <*fmt.wrapError | 0x35ac7e2dd140>: 
    image pull not confirmed; cause undetermined (insufficient evidence)
    polling observation: VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.004s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;
      - pod pull-secret-test container pull-test: ContainerCreating ()
    {
        msg: "image pull not confirmed; cause undetermined (insufficient evidence)\npolling observation: VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.004s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\n  - pod pull-secret-test container pull-test: ContainerCreating ()",
        err: <*fmt.wrapError | 0x35ac7e2dd120>{
            msg: "VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.004s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\n  - pod pull-secret-test container pull-test: ContainerCreating ()",
            err: <*errors.errorString | 0x35ac7fce7bf0>{
                s: "expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\n  - pod pull-secret-test container pull-test: ContainerCreating ()",
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret
Unexpected error:
    <*fmt.wrapError | 0x35ac7e2dd140>: 
    image pull not confirmed; cause undetermined (insufficient evidence)
    polling observation: VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.004s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;
      - pod pull-secret-test container pull-test: ContainerCreating ()
    {
        msg: "image pull not confirmed; cause undetermined (insufficient evidence)\npolling observation: VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.004s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\n  - pod pull-secret-test container pull-test: ContainerCreating ()",
        err: <*fmt.wrapError | 0x35ac7e2dd120>{
            msg: "VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.004s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\n  - pod pull-secret-test container pull-test: ContainerCreating ()",
            err: <*errors.errorString | 0x35ac7fce7bf0>{
                s: "expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\n  - pod pull-secret-test container pull-test: ContainerCreating ()",
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.39
Unexpected error:
    <*fmt.wrapError | 0xed2ec9e01c0>: 
    failed waiting for nodepool="npupgrade-4-20" in cluster="np-version-upgrade-cluster-989gmc" resourcegroup="rg-np-version-upgrade-989gmc-8hflngjf7pjq" to finish updating: GET https://rp.j9960704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/33eeff39-7b04-4af1-8912-9297ff607fdb
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/33eeff39-7b04-4af1-8912-9297ff607fdb",
      "name": "33eeff39-7b04-4af1-8912-9297ff607fdb",
      "status": "Failed",
      "startTime": "2026-10-22T16:42:14.996433941Z",
      "endTime": "2026-10-22T17:26:16.905655326Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool update did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is updating; [hypershiftNodePool] hypershift NodePool condition AllNodesHealthy is False: 1 of 4 machines are not healthy\nMachine h2l7v0j8s0t9z4z-npupgrade-4-20-b24jx-jpxdc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\n; hypershift NodePool condition AllMachinesReady is False: 1 of 4 machines are not ready\nMachine h2l7v0j8s0t9z4z-npupgrade-4-20-b24jx-jpxdc: Creating: virtualmachine creating or updating\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9960704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/33eeff39-7b04-4af1-8912-9297ff607fdb
            correlation ID: 49293550-ec1c-44c3-83a4-ef44b94443ce
            client request ID: 143bee29-fe61-4aa3-b798-569832148329
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.39
Unexpected error:
    <*fmt.wrapError | 0xed2ec9e01c0>: 
    failed waiting for nodepool="npupgrade-4-20" in cluster="np-version-upgrade-cluster-989gmc" resourcegroup="rg-np-version-upgrade-989gmc-8hflngjf7pjq" to finish updating: GET https://rp.j9960704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/33eeff39-7b04-4af1-8912-9297ff607fdb
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/33eeff39-7b04-4af1-8912-9297ff607fdb",
      "name": "33eeff39-7b04-4af1-8912-9297ff607fdb",
      "status": "Failed",
      "startTime": "2026-10-22T16:42:14.996433941Z",
      "endTime": "2026-10-22T17:26:16.905655326Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool update did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is updating; [hypershiftNodePool] hypershift NodePool condition AllNodesHealthy is False: 1 of 4 machines are not healthy\nMachine h2l7v0j8s0t9z4z-npupgrade-4-20-b24jx-jpxdc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\n; hypershift NodePool condition AllMachinesReady is False: 1 of 4 machines are not ready\nMachine h2l7v0j8s0t9z4z-npupgrade-4-20-b24jx-jpxdc: Creating: virtualmachine creating or updating\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9960704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/33eeff39-7b04-4af1-8912-9297ff607fdb
            correlation ID: 49293550-ec1c-44c3-83a4-ef44b94443ce
            client request ID: 143bee29-fe61-4aa3-b798-569832148329
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.
Expected success, but got an error:
    <*errors.errorString | 0xf9bfe384ae0>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8486, acm-agent=166
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8486, acm-agent=166",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.000s.
Expected success, but got an error:
    <*errors.errorString | 0xf9bfe384ae0>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8486, acm-agent=166
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8486, acm-agent=166",
    }

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-22T17:10:49Z
  Ended:       2026-10-22T17:16:51Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentRolloutStuck", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", condition="Progressing", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="progressdeadlineexceeded", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", status="false"
  Description: Rollout of deployment klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96/config-policy-controller is not progressing for longer than 30 minutes.

alert fired 35 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-22T16:41:51Z
  Ended:       2026-10-22T16:47:54Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-22T16:41:51Z
  Ended:       2026-10-22T16:48:49Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.65.78:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-r9dqq", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 3:
  State:       Resolved
  Started:     2026-10-22T16:41:51Z
  Ended:       2026-10-22T16:48:48Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.65.78:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-r9dqq", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 4:
  State:       Resolved
  Started:     2026-10-22T16:41:51Z
  Ended:       2026-10-22T16:47:55Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.65.78:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-r9dqq", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 5:
  State:       Resolved
  Started:     2026-10-22T16:41:51Z
  Ended:       2026-10-22T16:47:54Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.65.78:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-r9dqq", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 6:
  State:       Resolved
  Started:     2026-10-22T16:41:51Z
  Ended:       2026-10-22T16:48:48Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.65.78:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-r9dqq", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 7:
  State:       Resolved
  Started:     2026-10-22T16:41:51Z
  Ended:       2026-10-22T16:48:49Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.65.78:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-r9dqq", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjucsr4ans0dak8rtns8257caakk6v2/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 8:
  State:       Resolved
  Started:     2026-10-22T16:41:51Z
  Ended:       2026-10-22T16:48:48Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 9:
  State:       Resolved
  Started:     2026-10-22T16:42:48Z
  Ended:       2026-10-22T16:49:47Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 10:
  State:       Resolved
  Started:     2026-10-22T16:42:48Z
  Ended:       2026-10-22T16:48:48Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 11:
  State:       Resolved
  Started:     2026-10-22T16:42:48Z
  Ended:       2026-10-22T16:49:47Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 12:
  State:       Resolved
  Started:     2026-10-22T16:42:48Z
  Ended:       2026-10-22T16:49:47Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjudluq5d48hu4d9esi6v7ge1s36ua8/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 13:
  State:       Resolved
  Started:     2026-10-22T16:53:57Z
  Ended:       2026-10-22T16:59:50Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjuk4ke8vrnppho65r5l1u1hcmcf117", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjuk4ke8vrnppho65r5l1u1hcmcf117/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 14:
  State:       Resolved
  Started:     2026-10-22T16:56:49Z
  Ended:       2026-10-22T17:06:50Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 15:
  State:       Resolved
  Started:     2026-10-22T16:56:50Z
  Ended:       2026-10-22T17:06:50Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 16:
  State:       Resolved
  Started:     2026-10-22T16:56:50Z
  Ended:       2026-10-22T17:05:48Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 17:
  State:       Resolved
  Started:     2026-10-22T16:56:50Z
  Ended:       2026-10-22T17:05:48Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 18:
  State:       Resolved
  Started:     2026-10-22T16:56:50Z
  Ended:       2026-10-22T17:06:50Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 19:
  State:       Resolved
  Started:     2026-10-22T16:56:50Z
  Ended:       2026-10-22T17:06:51Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjulfqvt3bjekn98lrrcvta0oib3n1s/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 20:
  State:       Resolved
  Started:     2026-10-22T17:01:50Z
  Ended:       2026-10-22T17:16:50Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 21:
  State:       Resolved
  Started:     2026-10-22T17:01:50Z
  Ended:       2026-10-22T17:16:50Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 22:
  State:       Resolved
  Started:     2026-10-22T17:01:50Z
  Ended:       2026-10-22T17:16:50Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 23:
  State:       Resolved
  Started:     2026-10-22T17:01:50Z
  Ended:       2026-10-22T17:15:49Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 24:
  State:       Resolved
  Started:     2026-10-22T17:01:50Z
  Ended:       2026-10-22T17:15:49Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 25:
  State:       Resolved
  Started:     2026-10-22T17:01:50Z
  Ended:       2026-10-22T17:16:50Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjunb8atkidnq1m3miv0v20s57guh96/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 26:
  State:       Resolved
  Started:     2026-10-22T17:02:47Z
  Ended:       2026-10-22T17:14:50Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 27:
  State:       Resolved
  Started:     2026-10-22T17:02:47Z
  Ended:       2026-10-22T17:14:48Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 28:
  State:       Resolved
  Started:     2026-10-22T17:02:47Z
  Ended:       2026-10-22T17:12:48Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 29:
  State:       Resolved
  Started:     2026-10-22T17:02:47Z
  Ended:       2026-10-22T17:14:50Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 30:
  State:       Resolved
  Started:     2026-10-22T17:02:47Z
  Ended:       2026-10-22T17:12:48Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 31:
  State:       Resolved
  Started:     2026-10-22T17:02:47Z
  Ended:       2026-10-22T17:14:50Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjuo65073hh79n6dqmtmef62u80qgv9/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 32:
  State:       Resolved
  Started:     2026-10-22T17:03:49Z
  Ended:       2026-10-22T17:09:47Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjuojakjiaut5cjm0b9fprrjl0ljgun", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjuojakjiaut5cjm0b9fprrjl0ljgun/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 33:
  State:       Resolved
  Started:     2026-10-22T17:03:49Z
  Ended:       2026-10-22T17:09:48Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjuojakjiaut5cjm0b9fprrjl0ljgun", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjuojakjiaut5cjm0b9fprrjl0ljgun/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 34:
  State:       Resolved
  Started:     2026-10-22T17:03:49Z
  Ended:       2026-10-22T17:09:47Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjuojakjiaut5cjm0b9fprrjl0ljgun", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjuojakjiaut5cjm0b9fprrjl0ljgun/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 35:
  State:       Resolved
  Started:     2026-10-22T17:03:49Z
  Ended:       2026-10-22T17:09:48Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="klusterlet-2tjuojakjiaut5cjm0b9fprrjl0ljgun", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-x68b6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjuojakjiaut5cjm0b9fprrjl0ljgun/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.

alert fired 4 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-22T16:20:48Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7c96c6b89d-q84cd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="33074520-cd79-4e68-90bd-146270a8dbf6"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-q84cd (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-22T16:25:46Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j9064832-mgmt-1", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.65.78:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7c96c6b89d-dtrgd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="d46836a3-1737-4f96-a0d8-b3ef8af19f45"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-dtrgd (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-22T16:25:46Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j9064832-mgmt-2", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.65.59:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7c96c6b89d-q84cd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="33074520-cd79-4e68-90bd-146270a8dbf6"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-q84cd (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-22T16:28:47Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j9064832-mgmt-1", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.65.78:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9064832/providers/microsoft.monitor/accounts/services-j9064832", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7c96c6b89d-dtrgd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="d46836a3-1737-4f96-a0d8-b3ef8af19f45"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-dtrgd (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:179]: failed to capture node release images before upgrade
Unexpected error:
    <*fmt.wrapError | 0x390d19dbe00>: 
    list nodes (nodePool=npupgrade-4-20): Get "https://api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud:443/api/v1/nodes": dial tcp: lookup api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.233.194:52762->172.30.0.10:53: i/o timeout
    {
        msg: "list nodes (nodePool=npupgrade-4-20): Get \"https://api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud:443/api/v1/nodes\": dial tcp: lookup api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.233.194:52762->172.30.0.10:53: i/o timeout",
        err: <*url.Error | 0x390d1c076e0>{
            Op: "Get",
            URL: "https://api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud:443/api/v1/nodes",
            Err: <*net.OpError | 0x390d1655400>{
                Op: "dial",
                Net: "tcp",
                Source: nil,
                Addr: nil,
                Err: <*net.DNSError | 0x390d16552c0>{
                    UnwrapErr: nil,
                    Err: "read udp 172.24.233.194:52762->172.30.0.10:53: i/o timeout",
                    Name: "api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud",
                    Server: "172.30.0.10:53",
                    IsTimeout: true,
                    IsTemporary: true,
                    IsNotFound: false,
                },
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:179]: failed to capture node release images before upgrade
Unexpected error:
    <*fmt.wrapError | 0x390d19dbe00>: 
    list nodes (nodePool=npupgrade-4-20): Get "https://api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud:443/api/v1/nodes": dial tcp: lookup api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.233.194:52762->172.30.0.10:53: i/o timeout
    {
        msg: "list nodes (nodePool=npupgrade-4-20): Get \"https://api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud:443/api/v1/nodes\": dial tcp: lookup api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.233.194:52762->172.30.0.10:53: i/o timeout",
        err: <*url.Error | 0x390d1c076e0>{
            Op: "Get",
            URL: "https://api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud:443/api/v1/nodes",
            Err: <*net.OpError | 0x390d1655400>{
                Op: "dial",
                Net: "tcp",
                Source: nil,
                Addr: nil,
                Err: <*net.DNSError | 0x390d16552c0>{
                    UnwrapErr: nil,
                    Err: "read udp 172.24.233.194:52762->172.30.0.10:53: i/o timeout",
                    Name: "api.r5c4e4i1r9w0v2j.q1sr.j9370880.hcp.osadev.cloud",
                    Server: "172.30.0.10:53",
                    IsTimeout: true,
                    IsTemporary: true,
                    IsNotFound: false,
                },
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-22-h9tmsmbgdtrf/cluster-candidate-4-22-c8mmrm should provision
Unexpected error:
    <*fmt.wrapError | 0x1e5e1de42400>: 
    failed to create HCP cluster cluster-candidate-4-22-c8mmrm: failed waiting for cluster="cluster-candidate-4-22-c8mmrm" in resourcegroup="rg-candidate-4-22-h9tmsmbgdtrf" to finish creating: GET https://rp.j9370880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c4c9b08-e05d-4f74-a38c-fd407e61a545
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c4c9b08-e05d-4f74-a38c-fd407e61a545",
      "name": "3c4c9b08-e05d-4f74-a38c-fd407e61a545",
      "status": "Failed",
      "startTime": "2026-10-22T14:22:09.348889033Z",
      "endTime": "2026-10-22T14:51:16.010319539Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.19 is Partial (want Completed), started 21m32s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9370880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c4c9b08-e05d-4f74-a38c-fd407e61a545
            correlation ID: 5f95ddbb-902c-4016-8fa5-c1736f7561ec
            client request ID: 47f590fe-4c8d-4f3b-bde5-687e697310cd
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-22-h9tmsmbgdtrf/cluster-candidate-4-22-c8mmrm should provision
Unexpected error:
    <*fmt.wrapError | 0x1e5e1de42400>: 
    failed to create HCP cluster cluster-candidate-4-22-c8mmrm: failed waiting for cluster="cluster-candidate-4-22-c8mmrm" in resourcegroup="rg-candidate-4-22-h9tmsmbgdtrf" to finish creating: GET https://rp.j9370880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c4c9b08-e05d-4f74-a38c-fd407e61a545
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c4c9b08-e05d-4f74-a38c-fd407e61a545",
      "name": "3c4c9b08-e05d-4f74-a38c-fd407e61a545",
      "status": "Failed",
      "startTime": "2026-10-22T14:22:09.348889033Z",
      "endTime": "2026-10-22T14:51:16.010319539Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.19 is Partial (want Completed), started 21m32s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9370880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c4c9b08-e05d-4f74-a38c-fd407e61a545
            correlation ID: 5f95ddbb-902c-4016-8fa5-c1736f7561ec
            client request ID: 47f590fe-4c8d-4f3b-bde5-687e697310cd
occurred

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-22T14:37:59Z
  Ended:       2026-10-22T15:06:31Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j9370880-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tjss4leeh7ul4ugbdhur3figtfoc2ie-t8i0b6j2d1r3s2j", pod="router-85d447744d-ghchd", region="canadacentral", severity="warning"
  Description: Pod ocm-arohcpci01-2tjss4leeh7ul4ugbdhur3figtfoc2ie-t8i0b6j2d1r3s2j/router-85d447744d-ghchd has been in a non-ready state for longer than 5 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-23-bdn4zcvwkzhf/cluster-candidate-4-23-86tqbt should provision
Unexpected error:
    <*fmt.wrapError | 0x26f2228144e0>: 
    failed to create HCP cluster cluster-candidate-4-23-86tqbt: failed waiting for cluster="cluster-candidate-4-23-86tqbt" in resourcegroup="rg-candidate-4-23-bdn4zcvwkzhf" to finish creating: GET https://rp.j0492928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d723b871-78c8-4adc-9137-f20dd790b9ca
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d723b871-78c8-4adc-9137-f20dd790b9ca",
      "name": "d723b871-78c8-4adc-9137-f20dd790b9ca",
      "status": "Failed",
      "startTime": "2026-10-22T13:55:51.795775898Z",
      "endTime": "2026-10-22T14:24:54.794749941Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.23.0-ec.1 is Partial (want Completed), started 22m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0492928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d723b871-78c8-4adc-9137-f20dd790b9ca
            correlation ID: 134b5d7e-1756-4b7d-a04c-9d549c52db55
            client request ID: 73ca7188-a99d-4194-8ce5-029bf47125b1
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-23-bdn4zcvwkzhf/cluster-candidate-4-23-86tqbt should provision
Unexpected error:
    <*fmt.wrapError | 0x26f2228144e0>: 
    failed to create HCP cluster cluster-candidate-4-23-86tqbt: failed waiting for cluster="cluster-candidate-4-23-86tqbt" in resourcegroup="rg-candidate-4-23-bdn4zcvwkzhf" to finish creating: GET https://rp.j0492928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d723b871-78c8-4adc-9137-f20dd790b9ca
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d723b871-78c8-4adc-9137-f20dd790b9ca",
      "name": "d723b871-78c8-4adc-9137-f20dd790b9ca",
      "status": "Failed",
      "startTime": "2026-10-22T13:55:51.795775898Z",
      "endTime": "2026-10-22T14:24:54.794749941Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.23.0-ec.1 is Partial (want Completed), started 22m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j0492928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d723b871-78c8-4adc-9137-f20dd790b9ca
            correlation ID: 134b5d7e-1756-4b7d-a04c-9d549c52db55
            client request ID: 73ca7188-a99d-4194-8ce5-029bf47125b1
occurred

alert fired 19 time(s) (1 unknown, 18 known)
Firing 1:
  State:       Resolved
  Started:     2026-10-22T14:09:33Z
  Ended:       2026-10-22T14:38:33Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0492928-mgmt-1", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tjsfrjpd12fvkk41s1vdc57g83sl0d9-a8i0g4h2e2c7b8y", pod="router-6cb766858b-xgt9t", region="canadacentral", severity="warning"
  Description: Pod ocm-arohcpci01-2tjsfrjpd12fvkk41s1vdc57g83sl0d9-a8i0g4h2e2c7b8y/router-6cb766858b-xgt9t has been in a non-ready state for longer than 5 minutes.

time=2026-10-22T13:58:37.009Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 description="Step deploy-mce\n Kind: Helm\n"
time=2026-10-22T13:58:37.626Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 values="map[finalizeMceJobImage:mcr.microsoft.com/aks/command/runtime@sha256:37df117f1179d78fd372fd35878d53ff30dfb350fb571fb197f3a2bee48bd157 imageRegistry:arohcpsvcdev.azurecr.io imageRootRepository:acm-d-cache]"
time=2026-10-22T13:58:37.626Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1
time=2026-10-22T13:58:37.853Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 namespace=multicluster-engine
time=2026-10-22T13:58:38.094Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 dryRun=false
time=2026-10-22T13:58:38.094Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1
time=2026-10-22T13:58:38.280Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1
time=2026-10-22T14:03:45.649Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 err="resource Deployment/multicluster-engine/multicluster-engine-operator not ready. status: InProgress, message: Available: 1/2\ncontext deadline exceeded"
time=2026-10-22T14:03:45.650Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.ACM resourceGroup=management step=deploy-mce stamp=1 err="stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/multicluster-engine/multicluster-engine-operator not ready. status: InProgress, message: Available: 1/2\ncontext deadline exceeded"

alert fired 22 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:04:52Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0788224-mgmt-1", component="kubernetes-infrastructure", namespace="mgmt-agent", pod="mgmt-agent-54f9c547d8-j6wx5", region="westus3", severity="warning"
  Description: Pod mgmt-agent/mgmt-agent-54f9c547d8-j6wx5 has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:04:52Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0788224-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-kzbt8", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-kzbt8 has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:04:52Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0788224-mgmt-1", component="kubernetes-infrastructure", namespace="multicluster-engine", pod="multicluster-engine-operator-79c768d748-rrgxh", region="westus3", severity="warning"
  Description: Pod multicluster-engine/multicluster-engine-operator-79c768d748-rrgxh has been in a non-ready state for longer than 5 minutes.
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:04:52Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0788224-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-vmtzh", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-vmtzh has been in a non-ready state for longer than 5 minutes.
Firing 5:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:04:52Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0788224-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-lk66c", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-lk66c has been in a non-ready state for longer than 5 minutes.
Firing 6:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:04:53Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0788224-mgmt-1", component="kubernetes-infrastructure", namespace="kube-applier", pod="kube-applier-c5cf7d96-lrcd2", region="westus3", severity="warning"
  Description: Pod kube-applier/kube-applier-c5cf7d96-lrcd2 has been in a non-ready state for longer than 5 minutes.
Firing 7:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:04:53Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0788224-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-tp6wq", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-tp6wq has been in a non-ready state for longer than 5 minutes.
Firing 8:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:04:53Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0788224-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-gp8mb", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-gp8mb has been in a non-ready state for longer than 5 minutes.
Firing 9:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:04:53Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0788224-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-2qnxq", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-2qnxq has been in a non-ready state for longer than 5 minutes.
Firing 10:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:04:53Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0788224-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-8n4qd", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-8n4qd has been in a non-ready state for longer than 5 minutes.
Firing 11:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:04:53Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0788224-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-6vxsk", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-6vxsk has been in a non-ready state for longer than 5 minutes.
Firing 12:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:04:53Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0788224-mgmt-1", component="kubernetes-infrastructure", namespace="secret-sync-controller", pod="secrets-store-sync-controller-manager-7f4b9bc5df-h8csf", region="westus3", severity="warning"
  Description: Pod secret-sync-controller/secrets-store-sync-controller-manager-7f4b9bc5df-h8csf has been in a non-ready state for longer than 5 minutes.
Firing 13:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:04:53Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0788224-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-vgdgb", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-vgdgb has been in a non-ready state for longer than 5 minutes.
Firing 14:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:04:53Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0788224-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-kwkqc", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-kwkqc has been in a non-ready state for longer than 5 minutes.
Firing 15:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:04:53Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0788224-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-fh9lr", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-fh9lr has been in a non-ready state for longer than 5 minutes.
Firing 16:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:04:53Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0788224-svc", component="kubernetes-infrastructure", namespace="monitoring", pod="kube-events-67995d8d8b-jbwkn", region="westus3", severity="warning"
  Description: Pod monitoring/kube-events-67995d8d8b-jbwkn has been in a non-ready state for longer than 5 minutes.
Firing 17:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:05:51Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0788224-mgmt-1", component="kubernetes-infrastructure", namespace="multicluster-engine", pod="managedcluster-import-controller-v2-6cb46794f7-hqqwq", region="westus3", severity="warning"
  Description: Pod multicluster-engine/managedcluster-import-controller-v2-6cb46794f7-hqqwq has been in a non-ready state for longer than 5 minutes.
Firing 18:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:05:51Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0788224-mgmt-1", component="kubernetes-infrastructure", namespace="multicluster-engine", pod="ocm-webhook-68676899fd-srvpq", region="westus3", severity="warning"
  Description: Pod multicluster-engine/ocm-webhook-68676899fd-srvpq has been in a non-ready state for longer than 5 minutes.
Firing 19:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:05:51Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0788224-mgmt-1", component="kubernetes-infrastructure", namespace="multicluster-engine", pod="ocm-controller-859484fc49-hpp4f", region="westus3", severity="warning"
  Description: Pod multicluster-engine/ocm-controller-859484fc49-hpp4f has been in a non-ready state for longer than 5 minutes.
Firing 20:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:05:51Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0788224-mgmt-1", component="kubernetes-infrastructure", namespace="multicluster-engine", pod="cluster-manager-c5d6c587f-nf87h", region="westus3", severity="warning"
  Description: Pod multicluster-engine/cluster-manager-c5d6c587f-nf87h has been in a non-ready state for longer than 5 minutes.
Firing 21:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:05:51Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0788224-mgmt-1", component="kubernetes-infrastructure", namespace="multicluster-engine", pod="cluster-manager-c5d6c587f-xw9f9", region="westus3", severity="warning"
  Description: Pod multicluster-engine/cluster-manager-c5d6c587f-xw9f9 has been in a non-ready state for longer than 5 minutes.
Firing 22:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:05:51Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j0788224-mgmt-1", component="kubernetes-infrastructure", namespace="multicluster-engine", pod="clusterlifecycle-state-metrics-v2-7bb4664cf8-trs6j", region="westus3", severity="warning"
  Description: Pod multicluster-engine/clusterlifecycle-state-metrics-v2-7bb4664cf8-trs6j has been in a non-ready state for longer than 5 minutes.

alert fired 2 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:40:34Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentRolloutStuck", cluster="ci01-j9691264-mgmt-1", component="kubernetes-infrastructure", condition="Progressing", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.129:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264", namespace="open-cluster-management-agent-addon", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="progressdeadlineexceeded", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", status="false"
  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:41:38Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentRolloutStuck", cluster="ci01-j9691264-mgmt-1", component="kubernetes-infrastructure", condition="Progressing", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.129:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264", namespace="open-cluster-management-agent-addon", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="progressdeadlineexceeded", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", status="false"
  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.

alert fired 11 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:30:34Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9691264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.129:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264", namespace="open-cluster-management-agent-addon", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-22T14:31:34Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9691264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.129:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264", namespace="open-cluster-management-agent-addon", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 3:
  State:       Resolved
  Started:     2026-10-22T14:41:37Z
  Ended:       2026-10-22T14:47:33Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9691264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.129:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264", namespace="klusterlet-2tjsln0321cqtvn8430ocpgltipiguvb", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjsln0321cqtvn8430ocpgltipiguvb/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 4:
  State:       Resolved
  Started:     2026-10-22T14:41:37Z
  Ended:       2026-10-22T14:47:33Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9691264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.129:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264", namespace="klusterlet-2tjsln0321cqtvn8430ocpgltipiguvb", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjsln0321cqtvn8430ocpgltipiguvb/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 5:
  State:       Resolved
  Started:     2026-10-22T14:57:36Z
  Ended:       2026-10-22T15:06:35Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9691264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.129:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264", namespace="klusterlet-2tjsu1de5klk2l9lntbe3b6lp63khc9q", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjsu1de5klk2l9lntbe3b6lp63khc9q/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 6:
  State:       Resolved
  Started:     2026-10-22T14:58:40Z
  Ended:       2026-10-22T15:06:35Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9691264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.129:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264", namespace="klusterlet-2tjsu1de5klk2l9lntbe3b6lp63khc9q", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjsu1de5klk2l9lntbe3b6lp63khc9q/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 7:
  State:       Resolved
  Started:     2026-10-22T15:00:34Z
  Ended:       2026-10-22T15:06:34Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9691264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.129:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264", namespace="klusterlet-2tjsuurcgq00jdskheubnv1t28341vhu", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjsuurcgq00jdskheubnv1t28341vhu/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 8:
  State:       Resolved
  Started:     2026-10-22T15:01:34Z
  Ended:       2026-10-22T15:10:34Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9691264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.129:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264", namespace="klusterlet-2tjsvjhboesm03krfvrue116vlqlgtfk", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjsvjhboesm03krfvrue116vlqlgtfk/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 9:
  State:       Resolved
  Started:     2026-10-22T15:01:34Z
  Ended:       2026-10-22T15:10:34Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9691264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.129:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264", namespace="klusterlet-2tjsvjhboesm03krfvrue116vlqlgtfk", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjsvjhboesm03krfvrue116vlqlgtfk/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 10:
  State:       Resolved
  Started:     2026-10-22T15:03:36Z
  Ended:       2026-10-22T15:11:33Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9691264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.129:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264", namespace="klusterlet-2tjt0i26iaqpuk2cotnfqoqva7ea5k9q", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjt0i26iaqpuk2cotnfqoqva7ea5k9q/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 11:
  State:       Resolved
  Started:     2026-10-22T15:03:36Z
  Ended:       2026-10-22T15:11:33Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j9691264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.129:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9691264/providers/microsoft.monitor/accounts/services-j9691264", namespace="klusterlet-2tjt0i26iaqpuk2cotnfqoqva7ea5k9q", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-lwlkd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjt0i26iaqpuk2cotnfqoqva7ea5k9q/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.42
Unexpected error:
    <*fmt.wrapError | 0x2fdcc5c1a920>: 
    failed to create NodePool npnoedge-4-20: failed waiting for nodepool="npnoedge-4-20" for cluster "np-noedge-jhgf55" in resourcegroup="rg-np-noedge-jhgf55-jn7vd7sqrbwj" to finish creating: GET https://rp.j8698752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/053b6bb5-afd0-4ef3-ba63-cc853312d8d2
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/053b6bb5-afd0-4ef3-ba63-cc853312d8d2",
      "name": "053b6bb5-afd0-4ef3-ba63-cc853312d8d2",
      "status": "Failed",
      "startTime": "2026-10-22T14:32:42.03798521Z",
      "endTime": "2026-10-22T14:51:43.440263423Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine v6g6m6x8l5j1c5f-npnoedge-4-20-mrthg-lqgf9: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine v6g6m6x8l5j1c5f-npnoedge-4-20-mrthg-lqgf9: NotReady: * NodeHealthy:\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8698752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/053b6bb5-afd0-4ef3-ba63-cc853312d8d2
            correlation ID: 71832900-0fbb-4552-a679-c7de683bfa6e
            client request ID: 82d54b68-f5bf-46a3-9e65-10064ff56089
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.42
Unexpected error:
    <*fmt.wrapError | 0x2fdcc5c1a920>: 
    failed to create NodePool npnoedge-4-20: failed waiting for nodepool="npnoedge-4-20" for cluster "np-noedge-jhgf55" in resourcegroup="rg-np-noedge-jhgf55-jn7vd7sqrbwj" to finish creating: GET https://rp.j8698752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/053b6bb5-afd0-4ef3-ba63-cc853312d8d2
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/053b6bb5-afd0-4ef3-ba63-cc853312d8d2",
      "name": "053b6bb5-afd0-4ef3-ba63-cc853312d8d2",
      "status": "Failed",
      "startTime": "2026-10-22T14:32:42.03798521Z",
      "endTime": "2026-10-22T14:51:43.440263423Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine v6g6m6x8l5j1c5f-npnoedge-4-20-mrthg-lqgf9: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine v6g6m6x8l5j1c5f-npnoedge-4-20-mrthg-lqgf9: NotReady: * NodeHealthy:\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8698752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/053b6bb5-afd0-4ef3-ba63-cc853312d8d2
            correlation ID: 71832900-0fbb-4552-a679-c7de683bfa6e
            client request ID: 82d54b68-f5bf-46a3-9e65-10064ff56089
occurred

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x2a775c43e180>: 
    resource group "complex-cilium-kv-8hx8sgkq9px6": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cilium-cluster" in resourcegroup="complex-cilium-kv-8hx8sgkq9px6" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28cf5da8-4bf8-49ef-8a24-44a6ab872d73
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28cf5da8-4bf8-49ef-8a24-44a6ab872d73",
      "name": "28cf5da8-4bf8-49ef-8a24-44a6ab872d73",
      "status": "Failed",
      "startTime": "2026-10-22T14:21:07.679040667Z",
      "endTime": "2026-10-22T14:45:09.479893276Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjsi0li3c3ivjdhg5u6etupj5flrs9m still exists (deletion dispatched at 2026-10-22T14:21:07Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28cf5da8-4bf8-49ef-8a24-44a6ab872d73
            correlation ID: e13b5021-89b3-4639-913d-d5270a5dc946
            client request ID: 15895da4-10f4-47b9-b08f-39becbd952c2
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x2a775c43e180>: 
    resource group "complex-cilium-kv-8hx8sgkq9px6": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cilium-cluster" in resourcegroup="complex-cilium-kv-8hx8sgkq9px6" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28cf5da8-4bf8-49ef-8a24-44a6ab872d73
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28cf5da8-4bf8-49ef-8a24-44a6ab872d73",
      "name": "28cf5da8-4bf8-49ef-8a24-44a6ab872d73",
      "status": "Failed",
      "startTime": "2026-10-22T14:21:07.679040667Z",
      "endTime": "2026-10-22T14:45:09.479893276Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjsi0li3c3ivjdhg5u6etupj5flrs9m still exists (deletion dispatched at 2026-10-22T14:21:07Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/28cf5da8-4bf8-49ef-8a24-44a6ab872d73
            correlation ID: e13b5021-89b3-4639-913d-d5270a5dc946
            client request ID: 15895da4-10f4-47b9-b08f-39becbd952c2
occurred

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x27b60e3061f8>: 
    resource group "pullsecret-test-2qfk9tbgwjpl": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="pullsecret-hcp-cluster" in resourcegroup="pullsecret-test-2qfk9tbgwjpl" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0e34d898-c038-4aae-864f-7bc35e33d52f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0e34d898-c038-4aae-864f-7bc35e33d52f",
      "name": "0e34d898-c038-4aae-864f-7bc35e33d52f",
      "status": "Failed",
      "startTime": "2026-10-22T14:22:39.714077689Z",
      "endTime": "2026-10-22T14:46:39.512324074Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjsha6m9stttuajn7f257929ebjijis still exists (deletion dispatched at 2026-10-22T14:22:42Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0e34d898-c038-4aae-864f-7bc35e33d52f
            correlation ID: fdd89538-540d-4f74-94cb-acf48e5dcf5d
            client request ID: f1af71fe-def5-470f-a1f8-0006a3290e20
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x27b60e3061f8>: 
    resource group "pullsecret-test-2qfk9tbgwjpl": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="pullsecret-hcp-cluster" in resourcegroup="pullsecret-test-2qfk9tbgwjpl" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0e34d898-c038-4aae-864f-7bc35e33d52f
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0e34d898-c038-4aae-864f-7bc35e33d52f",
      "name": "0e34d898-c038-4aae-864f-7bc35e33d52f",
      "status": "Failed",
      "startTime": "2026-10-22T14:22:39.714077689Z",
      "endTime": "2026-10-22T14:46:39.512324074Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjsha6m9stttuajn7f257929ebjijis still exists (deletion dispatched at 2026-10-22T14:22:42Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0e34d898-c038-4aae-864f-7bc35e33d52f
            correlation ID: fdd89538-540d-4f74-94cb-acf48e5dcf5d
            client request ID: f1af71fe-def5-470f-a1f8-0006a3290e20
occurred

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x14030b5a2150>: 
    resource group "arm64-vm-cluster-mkmhd5qnk9lx": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="arm64-vm-hcp-cluster" in resourcegroup="arm64-vm-cluster-mkmhd5qnk9lx" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4a8e901d-725b-4e43-a6ab-53b69527c98c
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4a8e901d-725b-4e43-a6ab-53b69527c98c",
      "name": "4a8e901d-725b-4e43-a6ab-53b69527c98c",
      "status": "Failed",
      "startTime": "2026-10-22T14:16:46.278169996Z",
      "endTime": "2026-10-22T14:40:50.034223752Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjshbm80e288ae77lmgjasg7lrkdr39 still exists (deletion dispatched at 2026-10-22T14:16:46Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4a8e901d-725b-4e43-a6ab-53b69527c98c
            correlation ID: 75809158-6027-4075-b7d1-2543b49090e7
            client request ID: 76f25ad0-6660-420c-abae-8208cba3c40f
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x14030b5a2150>: 
    resource group "arm64-vm-cluster-mkmhd5qnk9lx": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="arm64-vm-hcp-cluster" in resourcegroup="arm64-vm-cluster-mkmhd5qnk9lx" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4a8e901d-725b-4e43-a6ab-53b69527c98c
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4a8e901d-725b-4e43-a6ab-53b69527c98c",
      "name": "4a8e901d-725b-4e43-a6ab-53b69527c98c",
      "status": "Failed",
      "startTime": "2026-10-22T14:16:46.278169996Z",
      "endTime": "2026-10-22T14:40:50.034223752Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjshbm80e288ae77lmgjasg7lrkdr39 still exists (deletion dispatched at 2026-10-22T14:16:46Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4a8e901d-725b-4e43-a6ab-53b69527c98c
            correlation ID: 75809158-6027-4075-b7d1-2543b49090e7
            client request ID: 76f25ad0-6660-420c-abae-8208cba3c40f
occurred

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x2ed9255f4018>: 
    resource group "rg-candidate-4-22-f565h2jdmqgh": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cluster-candidate-4-22-4j4s8h" in resourcegroup="rg-candidate-4-22-f565h2jdmqgh" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/837480d6-dd8d-4baa-bb95-0329fd032c3c
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/837480d6-dd8d-4baa-bb95-0329fd032c3c",
      "name": "837480d6-dd8d-4baa-bb95-0329fd032c3c",
      "status": "Failed",
      "startTime": "2026-10-22T14:27:18.140344478Z",
      "endTime": "2026-10-22T14:51:19.380950943Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjsha6vlktl4mo3ik8bbsikf69bgmlu still exists (deletion dispatched at 2026-10-22T14:27:21Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/837480d6-dd8d-4baa-bb95-0329fd032c3c
            correlation ID: c5cd1b4c-a04e-4f40-9b2c-1eb227504489
            client request ID: 3fa15ac4-aadc-4a06-b7db-144805d68e5c
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x2ed9255f4018>: 
    resource group "rg-candidate-4-22-f565h2jdmqgh": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cluster-candidate-4-22-4j4s8h" in resourcegroup="rg-candidate-4-22-f565h2jdmqgh" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/837480d6-dd8d-4baa-bb95-0329fd032c3c
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/837480d6-dd8d-4baa-bb95-0329fd032c3c",
      "name": "837480d6-dd8d-4baa-bb95-0329fd032c3c",
      "status": "Failed",
      "startTime": "2026-10-22T14:27:18.140344478Z",
      "endTime": "2026-10-22T14:51:19.380950943Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjsha6vlktl4mo3ik8bbsikf69bgmlu still exists (deletion dispatched at 2026-10-22T14:27:21Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/837480d6-dd8d-4baa-bb95-0329fd032c3c
            correlation ID: c5cd1b4c-a04e-4f40-9b2c-1eb227504489
            client request ID: 3fa15ac4-aadc-4a06-b7db-144805d68e5c
occurred

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x1661e2326120>: 
    resource group "external-auth-cluster-km6bz4ns8fzk": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="ea-cluster" in resourcegroup="external-auth-cluster-km6bz4ns8fzk" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc17306c-6d65-4b24-bd6f-a1392d064e72
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc17306c-6d65-4b24-bd6f-a1392d064e72",
      "name": "cc17306c-6d65-4b24-bd6f-a1392d064e72",
      "status": "Failed",
      "startTime": "2026-10-22T14:27:28.417107798Z",
      "endTime": "2026-10-22T14:51:29.402583281Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjshs28plfeftpnhlpn245qqpnk9ut6 still exists (deletion dispatched at 2026-10-22T14:27:29Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc17306c-6d65-4b24-bd6f-a1392d064e72
            correlation ID: 2d284687-9204-4517-b640-41ab53fbee0b
            client request ID: 9ef671a3-662c-492d-a9df-693795b1e91a
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x1661e2326120>: 
    resource group "external-auth-cluster-km6bz4ns8fzk": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="ea-cluster" in resourcegroup="external-auth-cluster-km6bz4ns8fzk" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc17306c-6d65-4b24-bd6f-a1392d064e72
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc17306c-6d65-4b24-bd6f-a1392d064e72",
      "name": "cc17306c-6d65-4b24-bd6f-a1392d064e72",
      "status": "Failed",
      "startTime": "2026-10-22T14:27:28.417107798Z",
      "endTime": "2026-10-22T14:51:29.402583281Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjshs28plfeftpnhlpn245qqpnk9ut6 still exists (deletion dispatched at 2026-10-22T14:27:29Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc17306c-6d65-4b24-bd6f-a1392d064e72
            correlation ID: 2d284687-9204-4517-b640-41ab53fbee0b
            client request ID: 9ef671a3-662c-492d-a9df-693795b1e91a
occurred

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x291accb9c480>: 
    resource group "rg-cluster-back-version-n5m774zrgzv9": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup="rg-cluster-back-version-n5m774zrgzv9", caused by: failed waiting for hcpCluster="cluster-ver-4-20" in resourcegroup="rg-cluster-back-version-n5m774zrgzv9" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster="cluster-ver-4-20" in resourcegroup="rg-cluster-back-version-n5m774zrgzv9" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded
    {
        errs: [
            <*fmt.wrapError | 0x291acc2cd520>{
                msg: "resource group \"rg-cluster-back-version-n5m774zrgzv9\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded",
                err: <*fmt.wrapError | 0x291acc2cd500>{
                    msg: "failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded",
                    err: <*fmt.wrapErrors | 0x291acd993a70>{
                        msg: "failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded",
                        errs: [
                            <*fmt.wrapErrors | 0x291acd993a40>{
                                msg: "failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded",
                                errs: [
                                    <*errors.errorString | 0x291acd69fed0>{
                                        s: "timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9",
                                    },
                                    <context.deadlineExceededError>{},
                                ],
                            },
                            <*fmt.wrapErrors | 0x291acd993a40>0x291acd993a40 (cyclic reference),
                        ],
                    },
                },
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x291accb9c480>: 
    resource group "rg-cluster-back-version-n5m774zrgzv9": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup="rg-cluster-back-version-n5m774zrgzv9", caused by: failed waiting for hcpCluster="cluster-ver-4-20" in resourcegroup="rg-cluster-back-version-n5m774zrgzv9" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster="cluster-ver-4-20" in resourcegroup="rg-cluster-back-version-n5m774zrgzv9" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded
    {
        errs: [
            <*fmt.wrapError | 0x291acc2cd520>{
                msg: "resource group \"rg-cluster-back-version-n5m774zrgzv9\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded",
                err: <*fmt.wrapError | 0x291acc2cd500>{
                    msg: "failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded",
                    err: <*fmt.wrapErrors | 0x291acd993a70>{
                        msg: "failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded",
                        errs: [
                            <*fmt.wrapErrors | 0x291acd993a40>{
                                msg: "failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-n5m774zrgzv9\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9, error: context deadline exceeded",
                                errs: [
                                    <*errors.errorString | 0x291acd69fed0>{
                                        s: "timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-n5m774zrgzv9",
                                    },
                                    <context.deadlineExceededError>{},
                                ],
                            },
                            <*fmt.wrapErrors | 0x291acd993a40>0x291acd993a40 (cyclic reference),
                        ],
                    },
                },
            },
        ],
    }
occurred

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x239068445020>: 
    resource group "cni-cilium-w6t5xjcmq5lh": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cilium-cl" in resourcegroup="cni-cilium-w6t5xjcmq5lh" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47d3a76f-90d6-4bf1-92be-1c3aeeccd29c
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47d3a76f-90d6-4bf1-92be-1c3aeeccd29c",
      "name": "47d3a76f-90d6-4bf1-92be-1c3aeeccd29c",
      "status": "Failed",
      "startTime": "2026-10-22T14:19:53.04024209Z",
      "endTime": "2026-10-22T14:43:59.86158375Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjshch4bfh839ftim6rnschfdun3698 still exists (deletion dispatched at 2026-10-22T14:19:53Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47d3a76f-90d6-4bf1-92be-1c3aeeccd29c
            correlation ID: 7aea69f5-9144-48be-be55-309aae2de7a4
            client request ID: db330e23-e554-4902-a60c-dc62e09132f3
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:
    <*errors.joinError | 0x239068445020>: 
    resource group "cni-cilium-w6t5xjcmq5lh": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster="cilium-cl" in resourcegroup="cni-cilium-w6t5xjcmq5lh" to finish deleting: GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47d3a76f-90d6-4bf1-92be-1c3aeeccd29c
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47d3a76f-90d6-4bf1-92be-1c3aeeccd29c",
      "name": "47d3a76f-90d6-4bf1-92be-1c3aeeccd29c",
      "status": "Failed",
      "startTime": "2026-10-22T14:19:53.04024209Z",
      "endTime": "2026-10-22T14:43:59.86158375Z",
      "error": {
        "code": "InternalServerError",
        "message": "cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tjshch4bfh839ftim6rnschfdun3698 still exists (deletion dispatched at 2026-10-22T14:19:53Z); [clusterServiceStatus] ClusterService state is \"uninstalling\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j4589440.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/47d3a76f-90d6-4bf1-92be-1c3aeeccd29c
            correlation ID: 7aea69f5-9144-48be-be55-309aae2de7a4
            client request ID: db330e23-e554-4902-a60c-dc62e09132f3
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1bbfb1338400>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-22T13:28:01.8728308Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-22T13:28:01.8728308Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1bbfb13383e0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-22T13:28:01.8728308Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1bbfb092c1e0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1bbfb13383c0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-22T13:28:01.8728308Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x1bbfb280d260>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x1bbfb1338400>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster"; let start = datetime(2026-10-22T13:28:01.8728308Z);
    let desires = cosmosResourceSnapshots
    | where resourceID startswith strcat(target, "/")
    | where resourceType =~ "microsoft.redhatopenshift/hcpopenshiftclusters/readdesires"
    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID
    | where isempty(content.deletionTimestamp)
    | extend manifest = content.properties.status.kubeContent
    | where manifest.kind == "HostedCluster" and isempty(manifest.metadata.deletionTimestamp)
    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),
        managementCluster = tolower(tostring(content.properties.spec.managementCluster))
    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)
    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, "-", hostedClusterName);
    let fleet = cosmosResourceSnapshots
    | where resourceType =~ "microsoft.redhatopenshift/stamps/managementclusters"
    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster
    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster
    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts
    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster
    | where isempty(content.deletionTimestamp)
    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), "/")[-1])
    | where isnotempty(clusterName)
    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;
    let hcp = desires
    | join kind=inner fleet on environment, region, sourceCluster, managementCluster
    | distinct environment, region, cluster, namespace;
    let inventory = materialize(kubernetesResourceSnapshots
    | where timestamp >= start and objectKind in ("Pod", "Deployment")
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name
    | where event != "Delete" and isempty(object.metadata.deletionTimestamp));
    let pods = inventory
    | where objectKind == "Pod" and object.metadata.labels.app == "private-router" and object.status.phase == "Running"
    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);
    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;
    let deployments = inventory
    | where objectKind == "Deployment" and name == "router"
    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);
    let summaries = containerLogs
    | where timestamp >= start and container_name == "swift-recorder"
    | where log.controller_name == "swift-router-check" and log.field == "summary"
    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)
    | lookup kind=inner hcp on environment, region, cluster, namespace
    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID
    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),
        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);
    hcp
    | join kind=leftouter deployments on environment, region, cluster, namespace
    | join kind=leftouter counts on environment, region, cluster, namespace
    | join kind=leftouter pods on environment, region, cluster, namespace
    | join kind=leftouter summaries on environment, region, cluster, namespace, UID
    | project Scope = strcat(environment, "/", region, "/", cluster, "/", namespace),
        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary
    | order by Scope asc, UID asc
    | take 101, Post "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query": context deadline exceeded
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-22T13:28:01.8728308Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
        err: <*fmt.wrapError | 0x1bbfb13383e0>{
            msg: "query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-22T13:28:01.8728308Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
            err: <*errors.Error | 0x1bbfb092c1e0>{
                Op: 1,
                Kind: 7,
                Err: <*fmt.wrapError | 0x1bbfb13383c0>{
                    msg: "With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mw8mkc7mm29l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-22T13:28:01.8728308Z);\nlet desires = cosmosResourceSnapshots\n| where resourceID startswith strcat(target, \"/\")\n| where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n| where isempty(content.deletionTimestamp)\n| extend manifest = content.properties.status.kubeContent\n| where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\nlet fleet = cosmosResourceSnapshots\n| where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n| where isempty(content.deletionTimestamp)\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n| where isnotempty(clusterName)\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\nlet hcp = desires\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\n| distinct environment, region, cluster, namespace;\nlet inventory = materialize(kubernetesResourceSnapshots\n| where timestamp >= start and objectKind in (\"Pod\", \"Deployment\")\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n| where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\nlet pods = inventory\n| where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\nlet deployments = inventory\n| where objectKind == \"Deployment\" and name == \"router\"\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\nlet summaries = containerLogs\n| where timestamp >= start and container_name == \"swift-recorder\"\n| where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n| lookup kind=inner hcp on environment, region, cluster, namespace\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\nhcp\n| join kind=leftouter deployments on environment, region, cluster, namespace\n| join kind=leftouter counts on environment, region, cluster, namespace\n| join kind=leftouter pods on environment, region, cluster, namespace\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\n| project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n| order by Scope asc, UID asc\n| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded",
                    err: <*url.Error | 0x1bbfb280d260>{
                        Op: "Post",
                        URL: "https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query",
                        Err: <context.deadlineExceededError>{},
                    },
                },
                restErrMsg: nil,
                decoded: nil,
                permanent: false,
                inner: nil,
            },
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool "np-2m9m9h" for cluster "cluster-candidate-4-21-2m9m9h"
Unexpected error:
    <*fmt.wrapError | 0x27893fb0e000>: 
    failed to create NodePool np-2m9m9h: failed waiting for nodepool="np-2m9m9h" for cluster "cluster-candidate-4-21-2m9m9h" in resourcegroup="rg-candidate-4-21-6lszx59zjddq" to finish creating: GET https://rp.j3313280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f57af8c9-c3cf-4aea-87b5-7b743fe62efb
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f57af8c9-c3cf-4aea-87b5-7b743fe62efb",
      "name": "f57af8c9-c3cf-4aea-87b5-7b743fe62efb",
      "status": "Failed",
      "startTime": "2026-10-22T13:54:06.562943785Z",
      "endTime": "2026-10-22T14:13:14.924879335Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine u8i0p8p0f8o9r7w-np-2m9m9h-w9cjd-9tqgw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-6lszx59zjddq-candidate-2m9m9h--managed/providers/Microsoft.Compute/virtualMachines/u8i0p8p0f8o9r7w-np-2m9m9h-w9cjd-9tqgw to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine u8i0p8p0f8o9r7w-np-2m9m9h-w9cjd-9tqgw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-6lszx59zjddq-candidate-2m9m9h--managed/providers/Microsoft.Compute/virtualMachines/u8i0p8p0f8o9r7w-np-2m9m9h-w9cjd-9tqgw to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3313280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f57af8c9-c3cf-4aea-87b5-7b743fe62efb
            correlation ID: b876de21-3542-4a1c-bc13-5b0b17d4888a
            client request ID: f374bcdf-2d88-4b72-b317-4d2ad1fe477a
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool "np-2m9m9h" for cluster "cluster-candidate-4-21-2m9m9h"
Unexpected error:
    <*fmt.wrapError | 0x27893fb0e000>: 
    failed to create NodePool np-2m9m9h: failed waiting for nodepool="np-2m9m9h" for cluster "cluster-candidate-4-21-2m9m9h" in resourcegroup="rg-candidate-4-21-6lszx59zjddq" to finish creating: GET https://rp.j3313280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f57af8c9-c3cf-4aea-87b5-7b743fe62efb
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f57af8c9-c3cf-4aea-87b5-7b743fe62efb",
      "name": "f57af8c9-c3cf-4aea-87b5-7b743fe62efb",
      "status": "Failed",
      "startTime": "2026-10-22T13:54:06.562943785Z",
      "endTime": "2026-10-22T14:13:14.924879335Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine u8i0p8p0f8o9r7w-np-2m9m9h-w9cjd-9tqgw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-6lszx59zjddq-candidate-2m9m9h--managed/providers/Microsoft.Compute/virtualMachines/u8i0p8p0f8o9r7w-np-2m9m9h-w9cjd-9tqgw to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine u8i0p8p0f8o9r7w-np-2m9m9h-w9cjd-9tqgw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-6lszx59zjddq-candidate-2m9m9h--managed/providers/Microsoft.Compute/virtualMachines/u8i0p8p0f8o9r7w-np-2m9m9h-w9cjd-9tqgw to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3313280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f57af8c9-c3cf-4aea-87b5-7b743fe62efb
            correlation ID: b876de21-3542-4a1c-bc13-5b0b17d4888a
            client request ID: f374bcdf-2d88-4b72-b317-4d2ad1fe477a
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster
Unexpected error:
    <*fmt.wrapError | 0x1971c8790ec0>: 
    failed waiting for nodepool="np-ver-4-20-32" for cluster "cluster-ver-4-20" in resourcegroup="rg-cluster-back-version-qg2q7d744n2t" to finish creating: context deadline exceeded
    {
        msg: "failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-qg2q7d744n2t\" to finish creating: context deadline exceeded",
        err: <context.deadlineExceededError>{},
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster
Unexpected error:
    <*fmt.wrapError | 0x1971c8790ec0>: 
    failed waiting for nodepool="np-ver-4-20-32" for cluster "cluster-ver-4-20" in resourcegroup="rg-cluster-back-version-qg2q7d744n2t" to finish creating: context deadline exceeded
    {
        msg: "failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-qg2q7d744n2t\" to finish creating: context deadline exceeded",
        err: <context.deadlineExceededError>{},
    }
occurred

time=2026-10-22T12:05:37.122Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 description="Step deploy\n Kind: Helm\n"
time=2026-10-22T12:05:37.861Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 values="map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:72abeb657cfbdd6a5891fc102a16429875d70c5932ad5994524d0246db914013 imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:55a0ad895a8f10193bb070883194449f5252c057353887af1ba0767760a0d2f0 imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:<nil>]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]"
time=2026-10-22T12:05:37.861Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2
time=2026-10-22T12:05:38.038Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 namespace=secret-sync-controller
time=2026-10-22T12:05:38.279Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 dryRun=false
time=2026-10-22T12:05:38.279Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2
time=2026-10-22T12:05:38.409Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2
time=2026-10-22T12:10:45.336Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err="resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\ncontext deadline exceeded"
time=2026-10-22T12:10:45.336Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err="stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\ncontext deadline exceeded"

alert fired 4 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-22T12:11:33Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4993536-mgmt-2", component="kubernetes-infrastructure", namespace="kube-applier", pod="kube-applier-695c66cfc4-vx26j", region="westus3", severity="warning"
  Description: Pod kube-applier/kube-applier-695c66cfc4-vx26j has been in a non-ready state for longer than 5 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-22T12:11:33Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4993536-mgmt-2", component="kubernetes-infrastructure", namespace="multicluster-engine", pod="multicluster-engine-operator-79c768d748-8qq8b", region="westus3", severity="warning"
  Description: Pod multicluster-engine/multicluster-engine-operator-79c768d748-8qq8b has been in a non-ready state for longer than 5 minutes.
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-22T12:11:33Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4993536-mgmt-2", component="kubernetes-infrastructure", namespace="mgmt-agent", pod="mgmt-agent-86dc97764f-kp7qm", region="westus3", severity="warning"
  Description: Pod mgmt-agent/mgmt-agent-86dc97764f-kp7qm has been in a non-ready state for longer than 5 minutes.
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-22T12:11:33Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j4993536-mgmt-2", component="kubernetes-infrastructure", namespace="secret-sync-controller", pod="secrets-store-sync-controller-manager-7f4b9bc5df-gtp8m", region="westus3", severity="warning"
  Description: Pod secret-sync-controller/secrets-store-sync-controller-manager-7f4b9bc5df-gtp8m has been in a non-ready state for longer than 5 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.004s.
Expected success, but got an error:
    <*errors.errorString | 0x3fcfb02035d0>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=10479, acm-agent=163
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=10479, acm-agent=163",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.004s.
Expected success, but got an error:
    <*errors.errorString | 0x3fcfb02035d0>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=10479, acm-agent=163
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=10479, acm-agent=163",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:122]: Timed out after 900.000s.
not all expected service metrics appeared in Azure Monitor
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:51 with:
expected maestro build info (MGMT) metrics but got no results (query: maestro_build_info)
Expected
    <[]prometheus.Result | len:0, cap:0>: []
not to be empty
fail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:122]: Timed out after 900.000s.
not all expected service metrics appeared in Azure Monitor
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:51 with:
expected maestro build info (MGMT) metrics but got no results (query: maestro_build_info)
Expected
    <[]prometheus.Result | len:0, cap:0>: []
not to be empty

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:149]: failed to verify simple web app runs on v20261001 cluster "v20261001"
Unexpected error:
    <*fmt.wrapError | 0xfc12762a440>: 
    DNS for route host agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0xfc127c420e0>{
            msg: "DNS for agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:149]: failed to verify simple web app runs on v20261001 cluster "v20261001"
Unexpected error:
    <*fmt.wrapError | 0xfc12762a440>: 
    DNS for route host agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
        err: <*fmt.wrapError | 0xfc127c420e0>{
            msg: "DNS for agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-bk2sz.apps.aro.v20261001.6kyw.j0641792.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:273]: Failed after 0.039s.
pod on cordoned node a2u9k2c9r3k4r5k-cordon-np-kf7tg-hqpg5 should remain Pending
The function passed to Consistently failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:269 with:
pod targeting cordoned node a2u9k2c9r3k4r5k-cordon-np-kf7tg-hqpg5 should not be assigned a node
Expected
    <string>: a2u9k2c9r3k4r5k-cordon-np-kf7tg-hqpg5
to be empty
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:273]: Failed after 0.039s.
pod on cordoned node a2u9k2c9r3k4r5k-cordon-np-kf7tg-hqpg5 should remain Pending
The function passed to Consistently failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:269 with:
pod targeting cordoned node a2u9k2c9r3k4r5k-cordon-np-kf7tg-hqpg5 should not be assigned a node
Expected
    <string>: a2u9k2c9r3k4r5k-cordon-np-kf7tg-hqpg5
to be empty

fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors
Unexpected error:
    <*fmt.wrapError | 0x268662498700>: 
    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster="idms-e2e-hcp-cluster" in resourcegroup="idms-zczv78nf9ddg" to finish creating: GET https://rp.j6096256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfac02ec-d27d-4ef0-b523-7cb0c0010faa
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InvalidResource
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfac02ec-d27d-4ef0-b523-7cb0c0010faa",
      "name": "dfac02ec-d27d-4ef0-b523-7cb0c0010faa",
      "status": "Failed",
      "startTime": "2026-10-22T11:20:30.318762088Z",
      "endTime": "2026-10-22T11:33:05.673152156Z",
      "error": {
        "code": "InvalidResource",
        "message": "ControlPlaneIdentitiesPermissionsClusterValidation: InternalError: Unable to verify control plane identities permissions."
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6096256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfac02ec-d27d-4ef0-b523-7cb0c0010faa
            correlation ID: f47f9f8e-d365-4823-b09b-4ccb1c4128f4
            client request ID: 32649701-fb7c-48c8-ac3b-ebfc52571839
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors
Unexpected error:
    <*fmt.wrapError | 0x268662498700>: 
    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster="idms-e2e-hcp-cluster" in resourcegroup="idms-zczv78nf9ddg" to finish creating: GET https://rp.j6096256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfac02ec-d27d-4ef0-b523-7cb0c0010faa
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InvalidResource
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfac02ec-d27d-4ef0-b523-7cb0c0010faa",
      "name": "dfac02ec-d27d-4ef0-b523-7cb0c0010faa",
      "status": "Failed",
      "startTime": "2026-10-22T11:20:30.318762088Z",
      "endTime": "2026-10-22T11:33:05.673152156Z",
      "error": {
        "code": "InvalidResource",
        "message": "ControlPlaneIdentitiesPermissionsClusterValidation: InternalError: Unable to verify control plane identities permissions."
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6096256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dfac02ec-d27d-4ef0-b523-7cb0c0010faa
            correlation ID: f47f9f8e-d365-4823-b09b-4ccb1c4128f4
            client request ID: 32649701-fb7c-48c8-ac3b-ebfc52571839
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.38
Unexpected error:
    <*fmt.wrapError | 0x2f3cf438a400>: 
    failed to create NodePool npupgrade-4-21: failed waiting for nodepool="npupgrade-4-21" for cluster "np-version-upgrade-cluster-2fr64m" in resourcegroup="rg-np-version-upgrade-2fr64m-zw6qzrcv9vrn" to finish creating: GET https://rp.j9390976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ee32fd12-b37a-4cd9-99a9-d6c6135b4269
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ee32fd12-b37a-4cd9-99a9-d6c6135b4269",
      "name": "ee32fd12-b37a-4cd9-99a9-d6c6135b4269",
      "status": "Failed",
      "startTime": "2026-10-22T11:18:21.118418049Z",
      "endTime": "2026-10-22T11:37:27.814332759Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine c5j8p1k5s3i9p3r-npupgrade-4-21-2t78b-vjsd2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-2fr64m-zw6qzrcv9vrn-np-upgrade-2f-dc28d34b/providers/Microsoft.Compute/virtualMachines/c5j8p1k5s3i9p3r-npupgrade-4-21-2t78b-vjsd2 to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine c5j8p1k5s3i9p3r-npupgrade-4-21-2t78b-vjsd2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-2fr64m-zw6qzrcv9vrn-np-upgrade-2f-dc28d34b/providers/Microsoft.Compute/virtualMachines/c5j8p1k5s3i9p3r-npupgrade-4-21-2t78b-vjsd2 to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9390976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ee32fd12-b37a-4cd9-99a9-d6c6135b4269
            correlation ID: 53e16191-0db4-40cf-9191-639da6db8ba5
            client request ID: ecd27427-b9cf-4b3a-aecb-2037c8f3702e
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.38
Unexpected error:
    <*fmt.wrapError | 0x2f3cf438a400>: 
    failed to create NodePool npupgrade-4-21: failed waiting for nodepool="npupgrade-4-21" for cluster "np-version-upgrade-cluster-2fr64m" in resourcegroup="rg-np-version-upgrade-2fr64m-zw6qzrcv9vrn" to finish creating: GET https://rp.j9390976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ee32fd12-b37a-4cd9-99a9-d6c6135b4269
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ee32fd12-b37a-4cd9-99a9-d6c6135b4269",
      "name": "ee32fd12-b37a-4cd9-99a9-d6c6135b4269",
      "status": "Failed",
      "startTime": "2026-10-22T11:18:21.118418049Z",
      "endTime": "2026-10-22T11:37:27.814332759Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine c5j8p1k5s3i9p3r-npupgrade-4-21-2t78b-vjsd2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-2fr64m-zw6qzrcv9vrn-np-upgrade-2f-dc28d34b/providers/Microsoft.Compute/virtualMachines/c5j8p1k5s3i9p3r-npupgrade-4-21-2t78b-vjsd2 to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine c5j8p1k5s3i9p3r-npupgrade-4-21-2t78b-vjsd2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-2fr64m-zw6qzrcv9vrn-np-upgrade-2f-dc28d34b/providers/Microsoft.Compute/virtualMachines/c5j8p1k5s3i9p3r-npupgrade-4-21-2t78b-vjsd2 to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j9390976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ee32fd12-b37a-4cd9-99a9-d6c6135b4269
            correlation ID: 53e16191-0db4-40cf-9191-639da6db8ba5
            client request ID: ecd27427-b9cf-4b3a-aecb-2037c8f3702e
occurred

alert fired 4 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-22T11:18:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j7460352-mgmt-2", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.205:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7f6f9c56c5-t67kq", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="d7472bae-c27c-4ab8-baa0-3a814678cda6"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-t67kq (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-22T11:21:48Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j7460352-mgmt-2", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.205:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7f6f9c56c5-t67kq", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="d7472bae-c27c-4ab8-baa0-3a814678cda6"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-t67kq (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-22T11:24:50Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7f6f9c56c5-lqfpq", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="e223dfef-7a26-486c-ad94-ac549a60abd4"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-lqfpq (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-22T11:25:46Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7f6f9c56c5-lqfpq", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="e223dfef-7a26-486c-ad94-ac549a60abd4"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-lqfpq (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").

alert fired 29 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-22T11:35:49Z
  Ended:       2026-10-22T11:41:52Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.205:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjpukl4clqjpol6nl56u5doppl4p8pn", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-q4hm8", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjpukl4clqjpol6nl56u5doppl4p8pn/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-22T11:35:49Z
  Ended:       2026-10-22T11:41:52Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.205:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjpukl4clqjpol6nl56u5doppl4p8pn", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-q4hm8", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjpukl4clqjpol6nl56u5doppl4p8pn/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 3:
  State:       Resolved
  Started:     2026-10-22T11:35:50Z
  Ended:       2026-10-22T11:41:52Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.205:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjpukl4clqjpol6nl56u5doppl4p8pn", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-q4hm8", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjpukl4clqjpol6nl56u5doppl4p8pn/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 4:
  State:       Resolved
  Started:     2026-10-22T11:35:50Z
  Ended:       2026-10-22T11:41:52Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjpuk6i2kb2t6qpc0cd60o1eif4bpro", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjpuk6i2kb2t6qpc0cd60o1eif4bpro/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 5:
  State:       Resolved
  Started:     2026-10-22T11:35:50Z
  Ended:       2026-10-22T11:41:51Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.205:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjpukl4clqjpol6nl56u5doppl4p8pn", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-q4hm8", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjpukl4clqjpol6nl56u5doppl4p8pn/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 6:
  State:       Resolved
  Started:     2026-10-22T11:53:53Z
  Ended:       2026-10-22T11:59:56Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq776rrupd6tlvsnjmtrb93odjkk8s", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq776rrupd6tlvsnjmtrb93odjkk8s/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 7:
  State:       Resolved
  Started:     2026-10-22T11:53:53Z
  Ended:       2026-10-22T12:00:52Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq776rrupd6tlvsnjmtrb93odjkk8s", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq776rrupd6tlvsnjmtrb93odjkk8s/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 8:
  State:       Resolved
  Started:     2026-10-22T11:53:53Z
  Ended:       2026-10-22T12:00:52Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq6p29ee4kfrd8338nnrlnus2g90cv", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq6p29ee4kfrd8338nnrlnus2g90cv/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 9:
  State:       Resolved
  Started:     2026-10-22T11:53:53Z
  Ended:       2026-10-22T12:00:52Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq776rrupd6tlvsnjmtrb93odjkk8s", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq776rrupd6tlvsnjmtrb93odjkk8s/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 10:
  State:       Resolved
  Started:     2026-10-22T11:53:53Z
  Ended:       2026-10-22T12:00:53Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq776rrupd6tlvsnjmtrb93odjkk8s", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq776rrupd6tlvsnjmtrb93odjkk8s/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 11:
  State:       Resolved
  Started:     2026-10-22T11:53:53Z
  Ended:       2026-10-22T12:00:52Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq776rrupd6tlvsnjmtrb93odjkk8s", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq776rrupd6tlvsnjmtrb93odjkk8s/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 12:
  State:       Resolved
  Started:     2026-10-22T11:54:49Z
  Ended:       2026-10-22T12:01:53Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq6p29ee4kfrd8338nnrlnus2g90cv", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq6p29ee4kfrd8338nnrlnus2g90cv/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 13:
  State:       Resolved
  Started:     2026-10-22T11:54:49Z
  Ended:       2026-10-22T12:01:53Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq6p29ee4kfrd8338nnrlnus2g90cv", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq6p29ee4kfrd8338nnrlnus2g90cv/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 14:
  State:       Resolved
  Started:     2026-10-22T11:54:49Z
  Ended:       2026-10-22T12:00:51Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq6p29ee4kfrd8338nnrlnus2g90cv", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq6p29ee4kfrd8338nnrlnus2g90cv/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 15:
  State:       Resolved
  Started:     2026-10-22T11:54:49Z
  Ended:       2026-10-22T12:01:53Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq6p29ee4kfrd8338nnrlnus2g90cv", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq6p29ee4kfrd8338nnrlnus2g90cv/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 16:
  State:       Resolved
  Started:     2026-10-22T11:55:52Z
  Ended:       2026-10-22T12:02:51Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 17:
  State:       Resolved
  Started:     2026-10-22T11:55:52Z
  Ended:       2026-10-22T12:04:54Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 18:
  State:       Resolved
  Started:     2026-10-22T11:55:52Z
  Ended:       2026-10-22T12:03:49Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 19:
  State:       Resolved
  Started:     2026-10-22T11:55:52Z
  Ended:       2026-10-22T12:02:51Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 20:
  State:       Resolved
  Started:     2026-10-22T11:56:51Z
  Ended:       2026-10-22T12:03:49Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 21:
  State:       Resolved
  Started:     2026-10-22T11:56:51Z
  Ended:       2026-10-22T12:04:54Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 22:
  State:       Resolved
  Started:     2026-10-22T11:56:51Z
  Ended:       2026-10-22T12:05:54Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 23:
  State:       Resolved
  Started:     2026-10-22T11:56:51Z
  Ended:       2026-10-22T12:04:55Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 24:
  State:       Resolved
  Started:     2026-10-22T11:56:51Z
  Ended:       2026-10-22T12:05:55Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 25:
  State:       Resolved
  Started:     2026-10-22T11:56:51Z
  Ended:       2026-10-22T12:05:55Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 26:
  State:       Resolved
  Started:     2026-10-22T11:56:51Z
  Ended:       2026-10-22T12:05:55Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq88pudougesdj4j66gmkg4ghnpiq4/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 27:
  State:       Resolved
  Started:     2026-10-22T11:56:51Z
  Ended:       2026-10-22T12:03:49Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq7sa7vem16l4k62h8lshdupmf0364/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 28:
  State:       Resolved
  Started:     2026-10-22T11:58:55Z
  Ended:       2026-10-22T12:04:54Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq9f63j3j2qj8oncuvau7tpnkhg8ub", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq9f63j3j2qj8oncuvau7tpnkhg8ub/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 29:
  State:       Resolved
  Started:     2026-10-22T11:58:55Z
  Ended:       2026-10-22T12:04:55Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7460352-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.115:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7460352/providers/microsoft.monitor/accounts/services-j7460352", namespace="klusterlet-2tjq9f63j3j2qj8oncuvau7tpnkhg8ub", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-rdjfd", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjq9f63j3j2qj8oncuvau7tpnkhg8ub/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.

time=2026-10-22T10:26:56.653Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description="Step istio-upgrade\n  Kind: IstioUpgrade\n  AKSCluster: {{ svc.aks.name }}\n"
time=2026-10-22T10:26:57.976Z level=INFO msg="Istio upgrade -- cluster state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8636416-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false
time=2026-10-22T10:26:57.976Z level=INFO msg="Upgrading to svc.istio.versions target" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8636416-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35
time=2026-10-22T10:26:58.804Z level=INFO msg="Istio upgrade -- mesh state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8636416-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2
time=2026-10-22T10:26:58.804Z level=INFO msg="Starting canary -- installing target alongside current" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8636416-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-22T10:26:58.804Z level=INFO msg="Starting ARM operation" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j8636416-svc
time=2026-10-22T10:42:01.308Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err="failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded"

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-23-464j6rgg5p8f/cluster-candidate-4-23-c98khv should provision
Unexpected error:
    <*fmt.wrapError | 0x15ffaf8e6ac0>: 
    failed to create HCP cluster cluster-candidate-4-23-c98khv: failed waiting for cluster="cluster-candidate-4-23-c98khv" in resourcegroup="rg-candidate-4-23-464j6rgg5p8f" to finish creating: GET https://rp.j7545856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/914ff296-32b8-41bd-9581-3a19f7c15e15
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/914ff296-32b8-41bd-9581-3a19f7c15e15",
      "name": "914ff296-32b8-41bd-9581-3a19f7c15e15",
      "status": "Failed",
      "startTime": "2026-10-22T09:57:41.668026777Z",
      "endTime": "2026-10-22T10:26:45.058545936Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7545856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/914ff296-32b8-41bd-9581-3a19f7c15e15
            correlation ID: ba6da990-0b2a-402c-b063-c501f6bad04e
            client request ID: 2565518b-4e23-4410-8a4e-8f021c515e5f
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-23-464j6rgg5p8f/cluster-candidate-4-23-c98khv should provision
Unexpected error:
    <*fmt.wrapError | 0x15ffaf8e6ac0>: 
    failed to create HCP cluster cluster-candidate-4-23-c98khv: failed waiting for cluster="cluster-candidate-4-23-c98khv" in resourcegroup="rg-candidate-4-23-464j6rgg5p8f" to finish creating: GET https://rp.j7545856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/914ff296-32b8-41bd-9581-3a19f7c15e15
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/914ff296-32b8-41bd-9581-3a19f7c15e15",
      "name": "914ff296-32b8-41bd-9581-3a19f7c15e15",
      "status": "Failed",
      "startTime": "2026-10-22T09:57:41.668026777Z",
      "endTime": "2026-10-22T10:26:45.058545936Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j7545856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/914ff296-32b8-41bd-9581-3a19f7c15e15
            correlation ID: ba6da990-0b2a-402c-b063-c501f6bad04e
            client request ID: 2565518b-4e23-4410-8a4e-8f021c515e5f
occurred

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-22T10:34:09Z
  Ended:       2026-10-22T10:47:10Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j7545856-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.184:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7545856/providers/microsoft.monitor/accounts/hcps-j7545856", namespace="ocm-arohcpci01-2tjp05s20i4o840anj97r3ots5bof56j-u5s2f5k1o3i2l8q", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-bxqfq", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjp05s20i4o840anj97r3ots5bof56j-u5s2f5k1o3i2l8q/capi-provider has not matched the expected number of replicas for longer than 30 minutes.

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-22T10:19:14Z
  Ended:       2026-10-22T10:39:09Z
  Severity:    Sev3
  Labels:      alertname="KubeStatefulSetReplicasMismatch", cluster="ci01-j7545856-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.184:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7545856/providers/microsoft.monitor/accounts/hcps-j7545856", namespace="ocm-arohcpci01-2tjp05s20i4o840anj97r3ots5bof56j-u5s2f5k1o3i2l8q", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-bxqfq", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", statefulset="etcd"
  Description: StatefulSet ocm-arohcpci01-2tjp05s20i4o840anj97r3ots5bof56j-u5s2f5k1o3i2l8q/etcd has not matched the expected number of replicas for longer than 15 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-22T10:19:14Z
  Ended:       2026-10-22T10:38:10Z
  Severity:    Sev3
  Labels:      alertname="KubeStatefulSetReplicasMismatch", cluster="ci01-j7545856-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.184:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7545856/providers/microsoft.monitor/accounts/hcps-j7545856", namespace="ocm-arohcpci01-2tjp05s20i4o840anj97r3ots5bof56j-u5s2f5k1o3i2l8q", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-bxqfq", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", statefulset="etcd"
  Description: StatefulSet ocm-arohcpci01-2tjp05s20i4o840anj97r3ots5bof56j-u5s2f5k1o3i2l8q/etcd has not matched the expected number of replicas for longer than 15 minutes.

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-22T10:28:07Z
  Ended:       2026-10-22T10:42:08Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j7545856-mgmt-1", component="kubernetes-infrastructure", container="etcd", endpoint="http", environment="ci01", instance="10.128.64.184:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7545856/providers/microsoft.monitor/accounts/hcps-j7545856", namespace="ocm-arohcpci01-2tjp05s20i4o840anj97r3ots5bof56j-u5s2f5k1o3i2l8q", pod="etcd-1", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="2bda07d3-27fb-4164-a4c3-6cfe1f72c141"
  Description: Pod ocm-arohcpci01-2tjp05s20i4o840anj97r3ots5bof56j-u5s2f5k1o3i2l8q/etcd-1 (etcd) is in waiting state (reason: "CrashLoopBackOff").
Firing 2:
  State:       Resolved
  Started:     2026-10-22T10:28:07Z
  Ended:       2026-10-22T10:42:08Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j7545856-mgmt-1", component="kubernetes-infrastructure", container="etcd", endpoint="http", environment="ci01", instance="10.128.64.184:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7545856/providers/microsoft.monitor/accounts/hcps-j7545856", namespace="ocm-arohcpci01-2tjp05s20i4o840anj97r3ots5bof56j-u5s2f5k1o3i2l8q", pod="etcd-1", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="2bda07d3-27fb-4164-a4c3-6cfe1f72c141"
  Description: Pod ocm-arohcpci01-2tjp05s20i4o840anj97r3ots5bof56j-u5s2f5k1o3i2l8q/etcd-1 (etcd) is in waiting state (reason: "CrashLoopBackOff").

alert fired 2 time(s) (1 unknown, 1 known)
Firing 1:
  State:       Resolved
  Started:     2026-10-22T09:11:20Z
  Ended:       2026-10-22T09:17:22Z
  Severity:    Sev3
  Labels:      alertname="KubePodNotReady", cluster="ci01-j2531968-mgmt-2", component="kubernetes-infrastructure", namespace="ocm-arohcpci01-2tjnp7iabfu8t4bg9odobu4iitn1rm2a-v20260901", pod="etcd-0", region="canadacentral", severity="warning"
  Description: Pod ocm-arohcpci01-2tjnp7iabfu8t4bg9odobu4iitn1rm2a-v20260901/etcd-0 has been in a non-ready state for longer than 5 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:361]: failed to create console OAuth client secret for external auth via VM
Unexpected error:
    <*errors.errorString | 0x339af03e78d0>: 
    timed out waiting for VM run command "e2e-runcommand-1792657320988467170" on VM "cidr-connectivity-test-test-vm"; canceled run command
    {
        s: "timed out waiting for VM run command \"e2e-runcommand-1792657320988467170\" on VM \"cidr-connectivity-test-test-vm\"; canceled run command",
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:361]: failed to create console OAuth client secret for external auth via VM
Unexpected error:
    <*errors.errorString | 0x339af03e78d0>: 
    timed out waiting for VM run command "e2e-runcommand-1792657320988467170" on VM "cidr-connectivity-test-test-vm"; canceled run command
    {
        s: "timed out waiting for VM run command \"e2e-runcommand-1792657320988467170\" on VM \"cidr-connectivity-test-test-vm\"; canceled run command",
    }
occurred

alert fired 30 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-22T08:33:21Z
  Ended:       2026-10-22T08:39:17Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.87:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjn8k6j55ugd7dnkujc325j0v4cs6pr", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-6tpfj", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjn8k6j55ugd7dnkujc325j0v4cs6pr/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-22T08:50:23Z
  Ended:       2026-10-22T08:56:20Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjnh74e75j44d0qjilt2dadh163si99", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjnh74e75j44d0qjilt2dadh163si99/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 3:
  State:       Resolved
  Started:     2026-10-22T08:50:24Z
  Ended:       2026-10-22T08:57:20Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjnh74e75j44d0qjilt2dadh163si99", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjnh74e75j44d0qjilt2dadh163si99/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 4:
  State:       Resolved
  Started:     2026-10-22T08:50:24Z
  Ended:       2026-10-22T08:57:20Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjnh74e75j44d0qjilt2dadh163si99", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjnh74e75j44d0qjilt2dadh163si99/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 5:
  State:       Resolved
  Started:     2026-10-22T08:51:19Z
  Ended:       2026-10-22T08:58:18Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 6:
  State:       Resolved
  Started:     2026-10-22T08:51:19Z
  Ended:       2026-10-22T08:57:20Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjnh74e75j44d0qjilt2dadh163si99", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjnh74e75j44d0qjilt2dadh163si99/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 7:
  State:       Resolved
  Started:     2026-10-22T08:51:19Z
  Ended:       2026-10-22T08:57:20Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjnh74e75j44d0qjilt2dadh163si99", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjnh74e75j44d0qjilt2dadh163si99/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 8:
  State:       Resolved
  Started:     2026-10-22T08:51:19Z
  Ended:       2026-10-22T08:57:20Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 9:
  State:       Resolved
  Started:     2026-10-22T08:51:19Z
  Ended:       2026-10-22T08:58:18Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 10:
  State:       Resolved
  Started:     2026-10-22T08:51:19Z
  Ended:       2026-10-22T08:59:18Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 11:
  State:       Resolved
  Started:     2026-10-22T08:51:19Z
  Ended:       2026-10-22T08:57:20Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 12:
  State:       Resolved
  Started:     2026-10-22T08:51:19Z
  Ended:       2026-10-22T08:58:18Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjnh3mq0mgu070r0ru5aaeudc1gmlni/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 13:
  State:       Resolved
  Started:     2026-10-22T08:53:18Z
  Ended:       2026-10-22T09:00:21Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 14:
  State:       Resolved
  Started:     2026-10-22T08:53:18Z
  Ended:       2026-10-22T09:01:18Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 15:
  State:       Resolved
  Started:     2026-10-22T08:53:18Z
  Ended:       2026-10-22T09:00:21Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 16:
  State:       Resolved
  Started:     2026-10-22T08:53:18Z
  Ended:       2026-10-22T09:05:18Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 17:
  State:       Resolved
  Started:     2026-10-22T08:53:18Z
  Ended:       2026-10-22T09:05:18Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 18:
  State:       Resolved
  Started:     2026-10-22T08:53:18Z
  Ended:       2026-10-22T09:04:20Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 19:
  State:       Resolved
  Started:     2026-10-22T08:53:18Z
  Ended:       2026-10-22T09:01:18Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 20:
  State:       Resolved
  Started:     2026-10-22T08:53:18Z
  Ended:       2026-10-22T09:05:18Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 21:
  State:       Resolved
  Started:     2026-10-22T08:53:18Z
  Ended:       2026-10-22T09:05:18Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 22:
  State:       Resolved
  Started:     2026-10-22T08:53:18Z
  Ended:       2026-10-22T09:04:19Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 23:
  State:       Resolved
  Started:     2026-10-22T08:53:18Z
  Ended:       2026-10-22T09:01:18Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 24:
  State:       Resolved
  Started:     2026-10-22T08:53:18Z
  Ended:       2026-10-22T09:04:19Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 25:
  State:       Resolved
  Started:     2026-10-22T08:53:18Z
  Ended:       2026-10-22T09:05:18Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 26:
  State:       Resolved
  Started:     2026-10-22T08:53:18Z
  Ended:       2026-10-22T09:01:18Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjni70h7vqbudvtmgk59hnqk3v51opo/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 27:
  State:       Resolved
  Started:     2026-10-22T08:53:18Z
  Ended:       2026-10-22T09:05:18Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 28:
  State:       Resolved
  Started:     2026-10-22T08:53:18Z
  Ended:       2026-10-22T09:05:18Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 29:
  State:       Resolved
  Started:     2026-10-22T08:53:18Z
  Ended:       2026-10-22T09:04:19Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjni184uhc3h89r653o27l5qb6h2smf/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 30:
  State:       Resolved
  Started:     2026-10-22T08:53:18Z
  Ended:       2026-10-22T09:05:18Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-tv7vp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjni2v392ng5edva32askfv0m7jtj7e/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.

alert fired 4 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-22T08:13:18Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7f6f9c56c5-2f299", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="f456e270-14f4-4101-89f7-ecf21d7b98ce"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-2f299 (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-22T08:15:15Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j2714240-mgmt-1", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.91:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7f6f9c56c5-2f299", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="f456e270-14f4-4101-89f7-ecf21d7b98ce"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-2f299 (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-22T08:16:17Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j2714240-mgmt-2", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.87:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7f6f9c56c5-8jwvp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="f1b0a0ab-3f08-4426-a498-92feced750e6"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-8jwvp (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-22T08:17:16Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j2714240-mgmt-2", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.87:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2714240/providers/microsoft.monitor/accounts/services-j2714240", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7f6f9c56c5-8jwvp", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="westus3", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="f1b0a0ab-3f08-4426-a498-92feced750e6"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-8jwvp (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.
Expected success, but got an error:
    <*errors.errorString | 0x291503b87830>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8179, acm-agent=225
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8179, acm-agent=225",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.
Expected success, but got an error:
    <*errors.errorString | 0x291503b87830>: 
    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8179, acm-agent=225
    {
        s: "missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=8179, acm-agent=225",
    }

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43
Unexpected error:
    <*fmt.wrapError | 0x1d7c44593440>: 
    failed to create NodePool npupgrade-4-20: failed waiting for nodepool="npupgrade-4-20" for cluster "np-version-upgrade-cluster-h4jhcq" in resourcegroup="rg-np-version-upgrade-h4jhcq-p4tvpw7x7ckb" to finish creating: GET https://rp.j5877888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b440666-a8bb-43ad-8b4a-822ed5350ff2
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b440666-a8bb-43ad-8b4a-822ed5350ff2",
      "name": "5b440666-a8bb-43ad-8b4a-822ed5350ff2",
      "status": "Failed",
      "startTime": "2026-10-22T08:17:17.757679885Z",
      "endTime": "2026-10-22T08:36:24.274326303Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine n1o4g3c1a5d2w5m-npupgrade-4-20-92gfm-7fbbt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-h4jhcq-p4tvpw7x7ckb-np-upgrade-h4-a1d46754/providers/Microsoft.Compute/virtualMachines/n1o4g3c1a5d2w5m-npupgrade-4-20-92gfm-7fbbt to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine n1o4g3c1a5d2w5m-npupgrade-4-20-92gfm-7fbbt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-h4jhcq-p4tvpw7x7ckb-np-upgrade-h4-a1d46754/providers/Microsoft.Compute/virtualMachines/n1o4g3c1a5d2w5m-npupgrade-4-20-92gfm-7fbbt to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5877888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b440666-a8bb-43ad-8b4a-822ed5350ff2
            correlation ID: 1da8ee73-93dd-4106-994f-5306d5b29df5
            client request ID: b60ab242-ab9c-4c04-b16f-c44b09af50eb
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43
Unexpected error:
    <*fmt.wrapError | 0x1d7c44593440>: 
    failed to create NodePool npupgrade-4-20: failed waiting for nodepool="npupgrade-4-20" for cluster "np-version-upgrade-cluster-h4jhcq" in resourcegroup="rg-np-version-upgrade-h4jhcq-p4tvpw7x7ckb" to finish creating: GET https://rp.j5877888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b440666-a8bb-43ad-8b4a-822ed5350ff2
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b440666-a8bb-43ad-8b4a-822ed5350ff2",
      "name": "5b440666-a8bb-43ad-8b4a-822ed5350ff2",
      "status": "Failed",
      "startTime": "2026-10-22T08:17:17.757679885Z",
      "endTime": "2026-10-22T08:36:24.274326303Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine n1o4g3c1a5d2w5m-npupgrade-4-20-92gfm-7fbbt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-h4jhcq-p4tvpw7x7ckb-np-upgrade-h4-a1d46754/providers/Microsoft.Compute/virtualMachines/n1o4g3c1a5d2w5m-npupgrade-4-20-92gfm-7fbbt to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine n1o4g3c1a5d2w5m-npupgrade-4-20-92gfm-7fbbt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-h4jhcq-p4tvpw7x7ckb-np-upgrade-h4-a1d46754/providers/Microsoft.Compute/virtualMachines/n1o4g3c1a5d2w5m-npupgrade-4-20-92gfm-7fbbt to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5877888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b440666-a8bb-43ad-8b4a-822ed5350ff2
            correlation ID: 1da8ee73-93dd-4106-994f-5306d5b29df5
            client request ID: b60ab242-ab9c-4c04-b16f-c44b09af50eb
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:122]: Timed out after 900.000s.
not all expected service metrics appeared in Azure Monitor
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:51 with:
expected maestro build info (MGMT) metrics but got no results (query: maestro_build_info)
Expected
    <[]prometheus.Result | len:0, cap:0>: []
not to be empty
fail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:122]: Timed out after 900.000s.
not all expected service metrics appeared in Azure Monitor
The function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:51 with:
expected maestro build info (MGMT) metrics but got no results (query: maestro_build_info)
Expected
    <[]prometheus.Result | len:0, cap:0>: []
not to be empty

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster
Unexpected error:
    <*fmt.wrapError | 0xed2a17d5560>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "pullsecret-hcp-cluster" in resourcegroup="pullsecret-test-kr8h8mgbckw7" to finish creating: GET https://rp.j5877888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a192dd85-8881-406d-a906-718d3d901c7e
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a192dd85-8881-406d-a906-718d3d901c7e",
      "name": "a192dd85-8881-406d-a906-718d3d901c7e",
      "status": "Failed",
      "startTime": "2026-10-22T08:02:04.742645812Z",
      "endTime": "2026-10-22T08:21:04.297195938Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-kr8h8mgbckw7--managed/providers/Microsoft.Compute/virtualMachines/n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-kr8h8mgbckw7--managed/n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-kr8h8mgbckw7--managed/providers/Microsoft.Network/networkInterfaces/n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5877888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a192dd85-8881-406d-a906-718d3d901c7e
            correlation ID: 72b107e0-3135-44c9-989b-a2246b2d1d75
            client request ID: db0337ce-fd31-490b-ad1f-f2e98766f20c
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster
Unexpected error:
    <*fmt.wrapError | 0xed2a17d5560>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "pullsecret-hcp-cluster" in resourcegroup="pullsecret-test-kr8h8mgbckw7" to finish creating: GET https://rp.j5877888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a192dd85-8881-406d-a906-718d3d901c7e
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a192dd85-8881-406d-a906-718d3d901c7e",
      "name": "a192dd85-8881-406d-a906-718d3d901c7e",
      "status": "Failed",
      "startTime": "2026-10-22T08:02:04.742645812Z",
      "endTime": "2026-10-22T08:21:04.297195938Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-kr8h8mgbckw7--managed/providers/Microsoft.Compute/virtualMachines/n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-kr8h8mgbckw7--managed/n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-kr8h8mgbckw7--managed/providers/Microsoft.Network/networkInterfaces/n5p3r7f1i1q6c6i-np-1-bpv7s-6dhmc-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5877888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a192dd85-8881-406d-a906-718d3d901c7e
            correlation ID: 72b107e0-3135-44c9-989b-a2246b2d1d75
            client request ID: db0337ce-fd31-490b-ad1f-f2e98766f20c
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster "cluster-candidate-5-1-8599c9"
Unexpected error:
    <*fmt.wrapError | 0x2e7ad2d862a0>: 
    DNS for route host agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded",
        err: <*fmt.wrapError | 0x2e7ad2a24480>{
            msg: "DNS for agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster "cluster-candidate-5-1-8599c9"
Unexpected error:
    <*fmt.wrapError | 0x2e7ad2d862a0>: 
    DNS for route host agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded",
        err: <*fmt.wrapError | 0x2e7ad2a24480>{
            msg: "DNS for agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8rlcw.apps.aro.q3t2a9x3p5m1g5y.vnpw.j3698176.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-0-8whfwd9s2cfc/cluster-candidate-5-0-rsdsmg should provision
Unexpected error:
    <*fmt.wrapError | 0x2868f20425c0>: 
    failed to create HCP cluster cluster-candidate-5-0-rsdsmg: failed waiting for cluster="cluster-candidate-5-0-rsdsmg" in resourcegroup="rg-candidate-5-0-8whfwd9s2cfc" to finish creating: GET https://rp.j3056640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/624deb2b-f6be-4ff9-9d3a-453b1d0fd058
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/624deb2b-f6be-4ff9-9d3a-453b1d0fd058",
      "name": "624deb2b-f6be-4ff9-9d3a-453b1d0fd058",
      "status": "Failed",
      "startTime": "2026-10-22T07:26:11.110423067Z",
      "endTime": "2026-10-22T07:55:18.619815497Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3056640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/624deb2b-f6be-4ff9-9d3a-453b1d0fd058
            correlation ID: 2a0d721c-fc9d-4caa-9738-de4581a559a8
            client request ID: 087da192-6efb-4007-8ca8-464d6f6bf167
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-5-0-8whfwd9s2cfc/cluster-candidate-5-0-rsdsmg should provision
Unexpected error:
    <*fmt.wrapError | 0x2868f20425c0>: 
    failed to create HCP cluster cluster-candidate-5-0-rsdsmg: failed waiting for cluster="cluster-candidate-5-0-rsdsmg" in resourcegroup="rg-candidate-5-0-8whfwd9s2cfc" to finish creating: GET https://rp.j3056640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/624deb2b-f6be-4ff9-9d3a-453b1d0fd058
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/624deb2b-f6be-4ff9-9d3a-453b1d0fd058",
      "name": "624deb2b-f6be-4ff9-9d3a-453b1d0fd058",
      "status": "Failed",
      "startTime": "2026-10-22T07:26:11.110423067Z",
      "endTime": "2026-10-22T07:55:18.619815497Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j3056640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/624deb2b-f6be-4ff9-9d3a-453b1d0fd058
            correlation ID: 2a0d721c-fc9d-4caa-9738-de4581a559a8
            client request ID: 087da192-6efb-4007-8ca8-464d6f6bf167
occurred

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-22T07:46:50Z
  Ended:       2026-10-22T08:14:51Z
  Severity:    Sev3
  Labels:      alertname="KubeStatefulSetReplicasMismatch", cluster="ci01-j3056640-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.145:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3056640/providers/microsoft.monitor/accounts/hcps-j3056640", namespace="ocm-arohcpci01-2tjmp4stnk12a380h2p81lrh4go8c0j3-c6o9t9a5x6k9a1t", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-f624w", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", statefulset="etcd"
  Description: StatefulSet ocm-arohcpci01-2tjmp4stnk12a380h2p81lrh4go8c0j3-c6o9t9a5x6k9a1t/etcd has not matched the expected number of replicas for longer than 15 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-22T07:46:50Z
  Ended:       2026-10-22T08:13:50Z
  Severity:    Sev3
  Labels:      alertname="KubeStatefulSetReplicasMismatch", cluster="ci01-j3056640-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", endpoint="http", environment="ci01", instance="10.128.64.145:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3056640/providers/microsoft.monitor/accounts/hcps-j3056640", namespace="ocm-arohcpci01-2tjmp4stnk12a380h2p81lrh4go8c0j3-c6o9t9a5x6k9a1t", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-f624w", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", statefulset="etcd"
  Description: StatefulSet ocm-arohcpci01-2tjmp4stnk12a380h2p81lrh4go8c0j3-c6o9t9a5x6k9a1t/etcd has not matched the expected number of replicas for longer than 15 minutes.

alert fired 2 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-22T07:48:50Z
  Ended:       2026-10-22T08:18:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j3056640-mgmt-1", component="kubernetes-infrastructure", container="etcd", endpoint="http", environment="ci01", instance="10.128.64.145:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3056640/providers/microsoft.monitor/accounts/hcps-j3056640", namespace="ocm-arohcpci01-2tjmp4stnk12a380h2p81lrh4go8c0j3-c6o9t9a5x6k9a1t", pod="etcd-1", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="f844d575-0b52-40fe-a2f1-220180987256"
  Description: Pod ocm-arohcpci01-2tjmp4stnk12a380h2p81lrh4go8c0j3-c6o9t9a5x6k9a1t/etcd-1 (etcd) is in waiting state (reason: "CrashLoopBackOff").
Firing 2:
  State:       Resolved
  Started:     2026-10-22T07:49:47Z
  Ended:       2026-10-22T08:18:49Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j3056640-mgmt-1", component="kubernetes-infrastructure", container="etcd", endpoint="http", environment="ci01", instance="10.128.64.145:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3056640/providers/microsoft.monitor/accounts/hcps-j3056640", namespace="ocm-arohcpci01-2tjmp4stnk12a380h2p81lrh4go8c0j3-c6o9t9a5x6k9a1t", pod="etcd-1", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="f844d575-0b52-40fe-a2f1-220180987256"
  Description: Pod ocm-arohcpci01-2tjmp4stnk12a380h2p81lrh4go8c0j3-c6o9t9a5x6k9a1t/etcd-1 (etcd) is in waiting state (reason: "CrashLoopBackOff").

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-22T08:01:49Z
  Ended:       2026-10-22T08:15:49Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j3056640-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="capi-provider", endpoint="http", environment="ci01", instance="10.128.64.145:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3056640/providers/microsoft.monitor/accounts/hcps-j3056640", namespace="ocm-arohcpci01-2tjmp4stnk12a380h2p81lrh4go8c0j3-c6o9t9a5x6k9a1t", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-f624w", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjmp4stnk12a380h2p81lrh4go8c0j3-c6o9t9a5x6k9a1t/capi-provider has not matched the expected number of replicas for longer than 30 minutes.

time=2026-10-22T06:22:04.877Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description="Step istio-upgrade\n  Kind: IstioUpgrade\n  AKSCluster: {{ svc.aks.name }}\n"
time=2026-10-22T06:22:06.224Z level=INFO msg="Istio upgrade -- cluster state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j7427584-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false
time=2026-10-22T06:22:06.224Z level=INFO msg="Upgrading to svc.istio.versions target" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j7427584-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35
time=2026-10-22T06:22:06.758Z level=INFO msg="Istio upgrade -- mesh state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j7427584-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2
time=2026-10-22T06:22:06.758Z level=INFO msg="Starting canary -- installing target alongside current" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j7427584-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-22T06:22:06.758Z level=INFO msg="Starting ARM operation" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j7427584-svc
time=2026-10-22T06:37:09.071Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err="failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded"

fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:136]: cluster viability check failed
Expected success, but got an error:
    <*errors.joinError | 0x347c86c470c8>: 
    VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1: Get "https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1": tls: failed to verify certificate: x509: certificate signed by unknown authority
    {
        errs: [
            <*fmt.wrapError | 0x347c84da9480>{
                msg: "VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority",
                err: <*errors.joinError | 0x347c85daeb28>{
                    errs: [
                        <*errors.errorString | 0x347c86a67600>{
                            s: "apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority",
                        },
                    ],
                },
            },
        ],
    }
fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:136]: cluster viability check failed
Expected success, but got an error:
    <*errors.joinError | 0x347c86c470c8>: 
    VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1: Get "https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1": tls: failed to verify certificate: x509: certificate signed by unknown authority
    {
        errs: [
            <*fmt.wrapError | 0x347c84da9480>{
                msg: "VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority",
                err: <*errors.joinError | 0x347c85daeb28>{
                    errs: [
                        <*errors.errorString | 0x347c86a67600>{
                            s: "apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.128.0.20:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority",
                        },
                    ],
                },
            },
        ],
    }

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool "np-56rfqn" for cluster "cluster-candidate-4-21-56rfqn"
Unexpected error:
    <*fmt.wrapError | 0x3a4be883de20>: 
    failed to create NodePool np-56rfqn: failed waiting for nodepool="np-56rfqn" for cluster "cluster-candidate-4-21-56rfqn" in resourcegroup="rg-candidate-4-21-72hp88nff97p" to finish creating: GET https://rp.j6553984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/689ced2e-06eb-4d9a-b1eb-1be22576e59e
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/689ced2e-06eb-4d9a-b1eb-1be22576e59e",
      "name": "689ced2e-06eb-4d9a-b1eb-1be22576e59e",
      "status": "Failed",
      "startTime": "2026-10-22T06:31:11.289499044Z",
      "endTime": "2026-10-22T06:50:16.978507942Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine k1a2h4t7l5j8y5h-np-56rfqn-z4897-vdfwj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-72hp88nff97p-candidate-56rfqn--managed/providers/Microsoft.Compute/virtualMachines/k1a2h4t7l5j8y5h-np-56rfqn-z4897-vdfwj to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine k1a2h4t7l5j8y5h-np-56rfqn-z4897-vdfwj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-72hp88nff97p-candidate-56rfqn--managed/providers/Microsoft.Compute/virtualMachines/k1a2h4t7l5j8y5h-np-56rfqn-z4897-vdfwj to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6553984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/689ced2e-06eb-4d9a-b1eb-1be22576e59e
            correlation ID: c789114c-8019-4481-bf73-689cc746416e
            client request ID: 963d7d6c-53b8-47ee-aceb-8ee1ac1d29bf
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool "np-56rfqn" for cluster "cluster-candidate-4-21-56rfqn"
Unexpected error:
    <*fmt.wrapError | 0x3a4be883de20>: 
    failed to create NodePool np-56rfqn: failed waiting for nodepool="np-56rfqn" for cluster "cluster-candidate-4-21-56rfqn" in resourcegroup="rg-candidate-4-21-72hp88nff97p" to finish creating: GET https://rp.j6553984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/689ced2e-06eb-4d9a-b1eb-1be22576e59e
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/689ced2e-06eb-4d9a-b1eb-1be22576e59e",
      "name": "689ced2e-06eb-4d9a-b1eb-1be22576e59e",
      "status": "Failed",
      "startTime": "2026-10-22T06:31:11.289499044Z",
      "endTime": "2026-10-22T06:50:16.978507942Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine k1a2h4t7l5j8y5h-np-56rfqn-z4897-vdfwj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-72hp88nff97p-candidate-56rfqn--managed/providers/Microsoft.Compute/virtualMachines/k1a2h4t7l5j8y5h-np-56rfqn-z4897-vdfwj to exist\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\nMachine k1a2h4t7l5j8y5h-np-56rfqn-z4897-vdfwj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-72hp88nff97p-candidate-56rfqn--managed/providers/Microsoft.Compute/virtualMachines/k1a2h4t7l5j8y5h-np-56rfqn-z4897-vdfwj to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6553984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/689ced2e-06eb-4d9a-b1eb-1be22576e59e
            correlation ID: c789114c-8019-4481-bf73-689cc746416e
            client request ID: 963d7d6c-53b8-47ee-aceb-8ee1ac1d29bf
occurred

alert fired 2 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:53:23Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="acm-agent", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/services-j2315264", namespace="klusterlet-2tjlnu5835h8mc45h83n6jc9202p3aa8", pod="klusterlet-addon-workmgr-6f457dbf58-2whgw", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="f288f60f-8a99-4810-a368-35076f7ec62b"
  Description: Pod klusterlet-2tjlnu5835h8mc45h83n6jc9202p3aa8/klusterlet-addon-workmgr-6f457dbf58-2whgw (acm-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:53:23Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="acm-agent", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/services-j2315264", namespace="klusterlet-2tjlnu5835h8mc45h83n6jc9202p3aa8", pod="klusterlet-addon-workmgr-6f457dbf58-2whgw", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="f288f60f-8a99-4810-a368-35076f7ec62b"
  Description: Pod klusterlet-2tjlnu5835h8mc45h83n6jc9202p3aa8/klusterlet-addon-workmgr-6f457dbf58-2whgw (acm-agent) is in waiting state (reason: "CrashLoopBackOff").

alert fired 72 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:53:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="azure-cloud-controller-manager", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf7u6ccqaq3v26qvoollh2q9l74n2-w4b2e7w9j1z2b4x", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf7u6ccqaq3v26qvoollh2q9l74n2-w4b2e7w9j1z2b4x/azure-cloud-controller-manager has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:53:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:53:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfcrbh5kspptntd809ena4hb9vfii-a9q5p6z0k8i2g7l", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfcrbh5kspptntd809ena4hb9vfii-a9q5p6z0k8i2g7l/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:53:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 5:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:53:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 6:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:53:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfcrbh5kspptntd809ena4hb9vfii-a9q5p6z0k8i2g7l", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfcrbh5kspptntd809ena4hb9vfii-a9q5p6z0k8i2g7l/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 7:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:54:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-state-metrics-hcp", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d/kube-state-metrics-hcp has not matched the expected number of replicas for longer than 30 minutes.
Firing 8:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:54:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-state-metrics-hcp", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d/kube-state-metrics-hcp has not matched the expected number of replicas for longer than 30 minutes.
Firing 9:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:54:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 10:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:54:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfbp2udn43p8trpi8pr6emnvn54vl-b5d0q1j3v5p7m5d/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 11:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:54:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="openshift-apiserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk/openshift-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 12:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:54:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 13:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:54:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfgobuhkqhcno47ho1a82btpgsopl-r2z9o3v2y6v9w2v", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfgobuhkqhcno47ho1a82btpgsopl-r2z9o3v2y6v9w2v/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 14:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:54:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf4p1e86tn9jn6bnmvi92avpsnmr2-t5t6z2c2m7l0h8n", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf4p1e86tn9jn6bnmvi92avpsnmr2-t5t6z2c2m7l0h8n/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 15:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:54:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 16:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:54:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfcrbh5kspptntd809ena4hb9vfii-a9q5p6z0k8i2g7l", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfcrbh5kspptntd809ena4hb9vfii-a9q5p6z0k8i2g7l/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 17:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:54:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf32j5gt6bsevjed4pcumqavc0dfc-a9v1e1y5n9p1r3w", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf32j5gt6bsevjed4pcumqavc0dfc-a9v1e1y5n9p1r3w/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 18:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:54:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfiphhlnq2mcs0l31tjg3idj77ah5-d1o8z7k7x5m2d6x", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfiphhlnq2mcs0l31tjg3idj77ah5-d1o8z7k7x5m2d6x/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 19:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:54:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfgobuhkqhcno47ho1a82btpgsopl-r2z9o3v2y6v9w2v", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfgobuhkqhcno47ho1a82btpgsopl-r2z9o3v2y6v9w2v/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 20:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:54:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf5gb21t62hgoa8sn8n9c4ioujgp3-k6f6b9p5q8z4t1j", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf5gb21t62hgoa8sn8n9c4ioujgp3-k6f6b9p5q8z4t1j/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 21:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:54:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf30b02hsqfk88ikscdflaqimt2mm-h9u6d9f1z0u8e8e", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf30b02hsqfk88ikscdflaqimt2mm-h9u6d9f1z0u8e8e/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 22:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:54:41Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfcrbh5kspptntd809ena4hb9vfii-a9q5p6z0k8i2g7l", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfcrbh5kspptntd809ena4hb9vfii-a9q5p6z0k8i2g7l/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 23:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf5hrm6lrr4j7n6nsn5n855udsr2j-d7v6t0q4c4t1b2j", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf5hrm6lrr4j7n6nsn5n855udsr2j-d7v6t0q4c4t1b2j/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 24:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf5gb21t62hgoa8sn8n9c4ioujgp3-k6f6b9p5q8z4t1j", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf5gb21t62hgoa8sn8n9c4ioujgp3-k6f6b9p5q8z4t1j/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 25:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlmpc8i7rs5966k2lohffos9lg7k7d-des-encrypt", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlmpc8i7rs5966k2lohffos9lg7k7d-des-encrypt/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 26:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlg9mro2kfb3e48neh4q78vgo8bi40-agg-cluster", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlg9mro2kfb3e48neh4q78vgo8bi40-agg-cluster/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 27:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf32j5gt6bsevjed4pcumqavc0dfc-a9v1e1y5n9p1r3w", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf32j5gt6bsevjed4pcumqavc0dfc-a9v1e1y5n9p1r3w/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 28:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf32j5gt6bsevjed4pcumqavc0dfc-a9v1e1y5n9p1r3w", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf32j5gt6bsevjed4pcumqavc0dfc-a9v1e1y5n9p1r3w/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 29:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf5gb21t62hgoa8sn8n9c4ioujgp3-k6f6b9p5q8z4t1j", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf5gb21t62hgoa8sn8n9c4ioujgp3-k6f6b9p5q8z4t1j/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 30:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf4p1e86tn9jn6bnmvi92avpsnmr2-t5t6z2c2m7l0h8n", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf4p1e86tn9jn6bnmvi92avpsnmr2-t5t6z2c2m7l0h8n/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 31:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf30b02hsqfk88ikscdflaqimt2mm-h9u6d9f1z0u8e8e", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf30b02hsqfk88ikscdflaqimt2mm-h9u6d9f1z0u8e8e/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 32:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 33:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf6q97hej2vqgr3kgdq6vf1n9nkh7-q2o2v7x7l2w1c9o", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf6q97hej2vqgr3kgdq6vf1n9nkh7-q2o2v7x7l2w1c9o/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 34:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf6q97hej2vqgr3kgdq6vf1n9nkh7-q2o2v7x7l2w1c9o", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf6q97hej2vqgr3kgdq6vf1n9nkh7-q2o2v7x7l2w1c9o/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 35:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="azure-cloud-controller-manager", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf5alfk8qk69tnqcis4t9ua0fv3bb-e6n0a3s6t4a0d8g", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf5alfk8qk69tnqcis4t9ua0fv3bb-e6n0a3s6t4a0d8g/azure-cloud-controller-manager has not matched the expected number of replicas for longer than 30 minutes.
Firing 36:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlg9mro2kfb3e48neh4q78vgo8bi40-agg-cluster", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlg9mro2kfb3e48neh4q78vgo8bi40-agg-cluster/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 37:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="azure-cloud-controller-manager", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf9pjn4g50ffukr1onbt218alfe4k-s5u2g5f8n8v7h1c", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf9pjn4g50ffukr1onbt218alfe4k-s5u2g5f8n8v7h1c/azure-cloud-controller-manager has not matched the expected number of replicas for longer than 30 minutes.
Firing 38:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="azure-cloud-controller-manager", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf9pjn4g50ffukr1onbt218alfe4k-s5u2g5f8n8v7h1c", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf9pjn4g50ffukr1onbt218alfe4k-s5u2g5f8n8v7h1c/azure-cloud-controller-manager has not matched the expected number of replicas for longer than 30 minutes.
Firing 39:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfgobuhkqhcno47ho1a82btpgsopl-r2z9o3v2y6v9w2v", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfgobuhkqhcno47ho1a82btpgsopl-r2z9o3v2y6v9w2v/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 40:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlmpc8i7rs5966k2lohffos9lg7k7d-des-encrypt", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlmpc8i7rs5966k2lohffos9lg7k7d-des-encrypt/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 41:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfgobuhkqhcno47ho1a82btpgsopl-r2z9o3v2y6v9w2v", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfgobuhkqhcno47ho1a82btpgsopl-r2z9o3v2y6v9w2v/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 42:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfiphhlnq2mcs0l31tjg3idj77ah5-d1o8z7k7x5m2d6x", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfiphhlnq2mcs0l31tjg3idj77ah5-d1o8z7k7x5m2d6x/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 43:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf30b02hsqfk88ikscdflaqimt2mm-h9u6d9f1z0u8e8e", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf30b02hsqfk88ikscdflaqimt2mm-h9u6d9f1z0u8e8e/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 44:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfiphhlnq2mcs0l31tjg3idj77ah5-d1o8z7k7x5m2d6x", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfiphhlnq2mcs0l31tjg3idj77ah5-d1o8z7k7x5m2d6x/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 45:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf30b02hsqfk88ikscdflaqimt2mm-h9u6d9f1z0u8e8e", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf30b02hsqfk88ikscdflaqimt2mm-h9u6d9f1z0u8e8e/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 46:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf5gb21t62hgoa8sn8n9c4ioujgp3-k6f6b9p5q8z4t1j", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf5gb21t62hgoa8sn8n9c4ioujgp3-k6f6b9p5q8z4t1j/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 47:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfao76s38o5i0d9igeopoi9khn4vg-x4d1m2h6y8p7d3y", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfao76s38o5i0d9igeopoi9khn4vg-x4d1m2h6y8p7d3y/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 48:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfao76s38o5i0d9igeopoi9khn4vg-x4d1m2h6y8p7d3y", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfao76s38o5i0d9igeopoi9khn4vg-x4d1m2h6y8p7d3y/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 49:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="openshift-apiserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf5k3ikf1giln0d9q4ntuokgl7tc8-ephemeral-disk/openshift-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 50:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf6q97hej2vqgr3kgdq6vf1n9nkh7-q2o2v7x7l2w1c9o", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf6q97hej2vqgr3kgdq6vf1n9nkh7-q2o2v7x7l2w1c9o/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 51:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="azure-cloud-controller-manager", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf5alfk8qk69tnqcis4t9ua0fv3bb-e6n0a3s6t4a0d8g", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf5alfk8qk69tnqcis4t9ua0fv3bb-e6n0a3s6t4a0d8g/azure-cloud-controller-manager has not matched the expected number of replicas for longer than 30 minutes.
Firing 52:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlmpc8i7rs5966k2lohffos9lg7k7d-des-encrypt", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlmpc8i7rs5966k2lohffos9lg7k7d-des-encrypt/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 53:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlmpc8i7rs5966k2lohffos9lg7k7d-des-encrypt", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlmpc8i7rs5966k2lohffos9lg7k7d-des-encrypt/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 54:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf5hrm6lrr4j7n6nsn5n855udsr2j-d7v6t0q4c4t1b2j", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf5hrm6lrr4j7n6nsn5n855udsr2j-d7v6t0q4c4t1b2j/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 55:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfiphhlnq2mcs0l31tjg3idj77ah5-d1o8z7k7x5m2d6x", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfiphhlnq2mcs0l31tjg3idj77ah5-d1o8z7k7x5m2d6x/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 56:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlg9mro2kfb3e48neh4q78vgo8bi40-agg-cluster", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlg9mro2kfb3e48neh4q78vgo8bi40-agg-cluster/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 57:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:46Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf9foan7mpm3sgb1bq64sqp3o92p2-r3b3j5q2z9b6m1f", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf9foan7mpm3sgb1bq64sqp3o92p2-r3b3j5q2z9b6m1f/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 58:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:46Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlg9mro2kfb3e48neh4q78vgo8bi40-agg-cluster", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlg9mro2kfb3e48neh4q78vgo8bi40-agg-cluster/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 59:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:46Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="openshift-apiserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlmpc8i7rs5966k2lohffos9lg7k7d-des-encrypt", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlmpc8i7rs5966k2lohffos9lg7k7d-des-encrypt/openshift-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 60:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:46Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf9foan7mpm3sgb1bq64sqp3o92p2-r3b3j5q2z9b6m1f", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf9foan7mpm3sgb1bq64sqp3o92p2-r3b3j5q2z9b6m1f/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 61:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:46Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf32j5gt6bsevjed4pcumqavc0dfc-a9v1e1y5n9p1r3w", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf32j5gt6bsevjed4pcumqavc0dfc-a9v1e1y5n9p1r3w/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 62:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:55:46Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf6q97hej2vqgr3kgdq6vf1n9nkh7-q2o2v7x7l2w1c9o", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf6q97hej2vqgr3kgdq6vf1n9nkh7-q2o2v7x7l2w1c9o/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 63:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:56:47Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="azure-cloud-controller-manager", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlg6hbvd4pqme7j4ns80o2aovvt54v-q4c2c9u5g9p5a2p", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlg6hbvd4pqme7j4ns80o2aovvt54v-q4c2c9u5g9p5a2p/azure-cloud-controller-manager has not matched the expected number of replicas for longer than 30 minutes.
Firing 64:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:56:47Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf7u6ccqaq3v26qvoollh2q9l74n2-w4b2e7w9j1z2b4x", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf7u6ccqaq3v26qvoollh2q9l74n2-w4b2e7w9j1z2b4x/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 65:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:56:47Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf9foan7mpm3sgb1bq64sqp3o92p2-r3b3j5q2z9b6m1f", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf9foan7mpm3sgb1bq64sqp3o92p2-r3b3j5q2z9b6m1f/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 66:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:56:47Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfao76s38o5i0d9igeopoi9khn4vg-x4d1m2h6y8p7d3y", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfao76s38o5i0d9igeopoi9khn4vg-x4d1m2h6y8p7d3y/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 67:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:56:47Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfa1kcsndl903g11sinfeoor1pae5-p5z7m8h6i6z2o6j", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfa1kcsndl903g11sinfeoor1pae5-p5z7m8h6i6z2o6j/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 68:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:56:47Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf9foan7mpm3sgb1bq64sqp3o92p2-r3b3j5q2z9b6m1f", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf9foan7mpm3sgb1bq64sqp3o92p2-r3b3j5q2z9b6m1f/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 69:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:56:47Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfa1kcsndl903g11sinfeoor1pae5-p5z7m8h6i6z2o6j", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfa1kcsndl903g11sinfeoor1pae5-p5z7m8h6i6z2o6j/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 70:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:56:47Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="packageserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlfao76s38o5i0d9igeopoi9khn4vg-x4d1m2h6y8p7d3y", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlfao76s38o5i0d9igeopoi9khn4vg-x4d1m2h6y8p7d3y/packageserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 71:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:56:47Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-apiserver", endpoint="http", environment="ci01", instance="10.128.65.62:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf2op037rdv3hdc8h1qto6lno6hal-u7g3d8u7r4l9m4n", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-5swl7", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf2op037rdv3hdc8h1qto6lno6hal-u7g3d8u7r4l9m4n/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.
Firing 72:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:56:47Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="kube-controller-manager", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/hcps-j2315264", namespace="ocm-arohcpci01-2tjlf32j5gt6bsevjed4pcumqavc0dfc-a9v1e1y5n9p1r3w", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment ocm-arohcpci01-2tjlf32j5gt6bsevjed4pcumqavc0dfc-a9v1e1y5n9p1r3w/kube-controller-manager has not matched the expected number of replicas for longer than 30 minutes.

alert fired 6 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:50:22Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/services-j2315264", namespace="klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:50:22Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/services-j2315264", namespace="klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:50:22Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/services-j2315264", namespace="klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:50:22Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/services-j2315264", namespace="klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 5:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:50:22Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/services-j2315264", namespace="klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 6:
  State:       Fired (not resolved)
  Started:     2026-10-22T06:50:22Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j2315264-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.191:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2315264/providers/microsoft.monitor/accounts/services-j2315264", namespace="klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-7lwj2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjlpeoinld44ovnvbco2pcb1sbmc79m/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints
Unexpected error:
    <*fmt.wrapError | 0x2e7f906828a0>: 
    failed waiting for nodepool="np-1" for cluster "cluster-np-labels-taints" in resourcegroup="rg-np-labels-taints-hlczr5rhlkx8" to finish creating: GET https://rp.j5246464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2d4aa15e-aedb-44e3-87cf-763478cb8fc1
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2d4aa15e-aedb-44e3-87cf-763478cb8fc1",
      "name": "2d4aa15e-aedb-44e3-87cf-763478cb8fc1",
      "status": "Failed",
      "startTime": "2026-10-22T06:07:20.975549407Z",
      "endTime": "2026-10-22T06:26:22.595569254Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-hlczr5rhlkx8--managed/providers/Microsoft.Compute/virtualMachines/d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t to exist\nMachine d1z2h7w9r6p6y8t-np-1-qmhgk-6rpkg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-hlczr5rhlkx8--managed/providers/Microsoft.Compute/virtualMachines/d1z2h7w9r6p6y8t-np-1-qmhgk-6rpkg to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t: Failed: interfaces failed to create or update. err: failed to get existing resource rg-np-labels-taints-hlczr5rhlkx8--managed/d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-hlczr5rhlkx8--managed/providers/Microsoft.Network/networkInterfaces/d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '3' seconds after additional tokens are available.... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5246464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2d4aa15e-aedb-44e3-87cf-763478cb8fc1
            correlation ID: 42569081-3a4c-40a5-ba81-59a78fa3c57e
            client request ID: 1bbe15c9-7618-4851-b4b5-17110a162bd6
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints
Unexpected error:
    <*fmt.wrapError | 0x2e7f906828a0>: 
    failed waiting for nodepool="np-1" for cluster "cluster-np-labels-taints" in resourcegroup="rg-np-labels-taints-hlczr5rhlkx8" to finish creating: GET https://rp.j5246464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2d4aa15e-aedb-44e3-87cf-763478cb8fc1
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2d4aa15e-aedb-44e3-87cf-763478cb8fc1",
      "name": "2d4aa15e-aedb-44e3-87cf-763478cb8fc1",
      "status": "Failed",
      "startTime": "2026-10-22T06:07:20.975549407Z",
      "endTime": "2026-10-22T06:26:22.595569254Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-hlczr5rhlkx8--managed/providers/Microsoft.Compute/virtualMachines/d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t to exist\nMachine d1z2h7w9r6p6y8t-np-1-qmhgk-6rpkg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-hlczr5rhlkx8--managed/providers/Microsoft.Compute/virtualMachines/d1z2h7w9r6p6y8t-np-1-qmhgk-6rpkg to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t: Failed: interfaces failed to create or update. err: failed to get existing resource rg-np-labels-taints-hlczr5rhlkx8--managed/d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-hlczr5rhlkx8--managed/providers/Microsoft.Network/networkInterfaces/d1z2h7w9r6p6y8t-np-1-qmhgk-2ck6t-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '3' seconds after additional tokens are available.... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5246464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2d4aa15e-aedb-44e3-87cf-763478cb8fc1
            correlation ID: 42569081-3a4c-40a5-ba81-59a78fa3c57e
            client request ID: 1bbe15c9-7618-4851-b4b5-17110a162bd6
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster "cluster-candidate-4-22-7lghhv"
Unexpected error:
    <*fmt.wrapError | 0x3dee4275c660>: 
    DNS for route host agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded",
        err: <*fmt.wrapError | 0x3dee4349c920>{
            msg: "DNS for agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:190]: failed to verify simple web app runs on cluster "cluster-candidate-4-22-7lghhv"
Unexpected error:
    <*fmt.wrapError | 0x3dee4275c660>: 
    DNS for route host agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded
    {
        msg: "DNS for route host agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded",
        err: <*fmt.wrapError | 0x3dee4349c920>{
            msg: "DNS for agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8z9nr.apps.aro.r6z2d3l5y4t7l0b.r2ri.j6057856.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded",
            err: <context.deadlineExceededError>{},
        },
    }
occurred

time=2026-10-22T05:26:16.605Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description="Step istio-upgrade\n  Kind: IstioUpgrade\n  AKSCluster: {{ svc.aks.name }}\n"
time=2026-10-22T05:26:17.883Z level=INFO msg="Istio upgrade -- cluster state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false
time=2026-10-22T05:26:17.883Z level=INFO msg="Upgrading to svc.istio.versions target" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35
time=2026-10-22T05:26:18.860Z level=INFO msg="Istio upgrade -- mesh state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2
time=2026-10-22T05:26:18.860Z level=INFO msg="Starting canary -- installing target alongside current" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-22T05:26:18.860Z level=INFO msg="Starting ARM operation" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j9350144-svc
time=2026-10-22T05:29:23.971Z level=INFO msg="Cluster provisioning state" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade state=Succeeded
time=2026-10-22T05:29:23.971Z level=INFO msg="ARM operation complete" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary elapsed=3m5s
time=2026-10-22T05:29:23.971Z level=INFO msg="Step 1/9: Ensuring MISE ext-authz ConfigMap" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-22T05:29:24.574Z level=INFO msg="ConfigMap created" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade/revision-configmap name=istio-shared-configmap-asm-1-30
time=2026-10-22T05:29:24.574Z level=INFO msg="Step 2/9: Checking tag-based namespace safety" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-22T05:29:24.574Z level=INFO msg="Step 3/9: Verifying control plane health and flipping tag webhook" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-22T05:29:25.174Z level=INFO msg="Created revision tag webhook" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade/revision-tag webhook=istio-revision-tag-prod-stable-aks-istio-system revision=asm-1-30
time=2026-10-22T05:29:25.174Z level=INFO msg="Step 4/9: Ensuring ingress gateway annotations" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-22T05:29:25.205Z level=INFO msg="Applying annotations" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade/ingress-annotations service=aks-istio-ingressgateway-external
time=2026-10-22T05:29:25.415Z level=INFO msg="Step 5/9: Migrating workloads to target revision" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-22T05:29:25.470Z level=INFO msg="Step 6/9: Running health check" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-22T05:29:25.913Z level=INFO msg="Health check passed -- checking for orphaned workloads before completing canary" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-22T05:29:25.913Z level=INFO msg="Step 7/9: Running orphan guard" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-22T05:29:25.913Z level=INFO msg="No orphaned workloads -- completing canary" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-22T05:29:25.913Z level=INFO msg="Step 8/9: Completing canary via ARM (point of no return)" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9350144-svc versions=asm-1-30 logger=istio-upgrade
time=2026-10-22T05:29:25.913Z level=INFO msg="Starting ARM operation" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=complete-canary cluster=ci01-j9350144-svc
time=2026-10-22T05:44:28.867Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err="failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded"

fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool "np-jbvlzr" for cluster "cluster-candidate-5-1-jbvlzr"
Unexpected error:
    <*fmt.wrapError | 0x16e3bf104600>: 
    failed to create NodePool np-jbvlzr: failed waiting for nodepool="np-jbvlzr" for cluster "cluster-candidate-5-1-jbvlzr" in resourcegroup="rg-candidate-5-1-jr4zj7tw52p6" to finish creating: GET https://rp.j6940672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/905e6e4d-deed-4c98-88d4-7e7d915bea9c
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/905e6e4d-deed-4c98-88d4-7e7d915bea9c",
      "name": "905e6e4d-deed-4c98-88d4-7e7d915bea9c",
      "status": "Failed",
      "startTime": "2026-10-22T03:36:41.547739466Z",
      "endTime": "2026-10-22T03:55:45.436892812Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-krx2m: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-95drd: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-5-1-jr4zj7tw52p6-candidate-jbvlzr--managed/g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-95drd-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-jr4zj7tw52p6-candidate-jbvlzr--managed/providers/Microsoft.Network/networkInterfaces/g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-95drd-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5157a63c-87dc-4680-8f46-954b46399bdc' exceeded. Please try again after '10' se... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6940672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/905e6e4d-deed-4c98-88d4-7e7d915bea9c
            correlation ID: 0f6414e6-d1c6-4b3c-b1b7-40aee43cce2b
            client request ID: 57e0289e-c2ad-44a4-9341-dc67578bae73
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool "np-jbvlzr" for cluster "cluster-candidate-5-1-jbvlzr"
Unexpected error:
    <*fmt.wrapError | 0x16e3bf104600>: 
    failed to create NodePool np-jbvlzr: failed waiting for nodepool="np-jbvlzr" for cluster "cluster-candidate-5-1-jbvlzr" in resourcegroup="rg-candidate-5-1-jr4zj7tw52p6" to finish creating: GET https://rp.j6940672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/905e6e4d-deed-4c98-88d4-7e7d915bea9c
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/905e6e4d-deed-4c98-88d4-7e7d915bea9c",
      "name": "905e6e4d-deed-4c98-88d4-7e7d915bea9c",
      "status": "Failed",
      "startTime": "2026-10-22T03:36:41.547739466Z",
      "endTime": "2026-10-22T03:55:45.436892812Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\nMachine g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-krx2m: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-95drd: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-5-1-jr4zj7tw52p6-candidate-jbvlzr--managed/g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-95drd-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-jr4zj7tw52p6-candidate-jbvlzr--managed/providers/Microsoft.Network/networkInterfaces/g5o8j6w4a0x6e0p-np-jbvlzr-xq2nz-95drd-nic\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5157a63c-87dc-4680-8f46-954b46399bdc' exceeded. Please try again after '10' se... message truncated\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j6940672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/905e6e4d-deed-4c98-88d4-7e7d915bea9c
            correlation ID: 0f6414e6-d1c6-4b3c-b1b7-40aee43cce2b
            client request ID: 57e0289e-c2ad-44a4-9341-dc67578bae73
occurred

alert fired 4 time(s)
Firing 1:
  State:       Fired (not resolved)
  Started:     2026-10-22T03:42:45Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7f6f9c56c5-b8qtg", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="5239caaa-164a-4f5a-831a-167d2502f038"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-b8qtg (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 2:
  State:       Fired (not resolved)
  Started:     2026-10-22T03:42:46Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j6940672-mgmt-2", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.65.63:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7f6f9c56c5-vdkk2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="500be196-af9b-4978-b1e8-f05efb43b0f6"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-vdkk2 (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 3:
  State:       Fired (not resolved)
  Started:     2026-10-22T03:42:46Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j6940672-mgmt-2", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.65.63:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7f6f9c56c5-vdkk2", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="500be196-af9b-4978-b1e8-f05efb43b0f6"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-vdkk2 (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").
Firing 4:
  State:       Fired (not resolved)
  Started:     2026-10-22T03:42:46Z
  Severity:    Sev3
  Labels:      alertname="KubePodCrashLooping", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="hypershift-addon-agent", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="open-cluster-management-agent-addon", pod="hypershift-addon-agent-7f6f9c56c5-b8qtg", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", reason="CrashLoopBackOff", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning", uid="5239caaa-164a-4f5a-831a-167d2502f038"
  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-b8qtg (hypershift-addon-agent) is in waiting state (reason: "CrashLoopBackOff").

alert fired 30 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-22T03:58:47Z
  Ended:       2026-10-22T04:06:48Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 2:
  State:       Resolved
  Started:     2026-10-22T03:58:47Z
  Ended:       2026-10-22T04:06:48Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 3:
  State:       Resolved
  Started:     2026-10-22T03:58:47Z
  Ended:       2026-10-22T04:04:47Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 4:
  State:       Resolved
  Started:     2026-10-22T03:58:47Z
  Ended:       2026-10-22T04:05:48Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 5:
  State:       Resolved
  Started:     2026-10-22T03:58:47Z
  Ended:       2026-10-22T04:05:48Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 6:
  State:       Resolved
  Started:     2026-10-22T03:58:47Z
  Ended:       2026-10-22T04:04:47Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjj8378cqgbrqc6c6582f25i8ea5252/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 7:
  State:       Resolved
  Started:     2026-10-22T04:00:47Z
  Ended:       2026-10-22T04:06:48Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-2", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.65.63:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjj81ietnv2jbgeg13jpmpsis0ij8qo", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-67xzr", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjj81ietnv2jbgeg13jpmpsis0ij8qo/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 8:
  State:       Resolved
  Started:     2026-10-22T04:17:44Z
  Ended:       2026-10-22T04:25:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjjhu9ej7evo873ckvf6h944dkcatga", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjjhu9ej7evo873ckvf6h944dkcatga/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 9:
  State:       Resolved
  Started:     2026-10-22T04:17:44Z
  Ended:       2026-10-22T04:28:48Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 10:
  State:       Resolved
  Started:     2026-10-22T04:17:44Z
  Ended:       2026-10-22T04:25:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjjhu9ej7evo873ckvf6h944dkcatga", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjjhu9ej7evo873ckvf6h944dkcatga/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 11:
  State:       Resolved
  Started:     2026-10-22T04:17:44Z
  Ended:       2026-10-22T04:30:46Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 12:
  State:       Resolved
  Started:     2026-10-22T04:17:44Z
  Ended:       2026-10-22T04:24:47Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjjhu9ej7evo873ckvf6h944dkcatga", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjjhu9ej7evo873ckvf6h944dkcatga/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 13:
  State:       Resolved
  Started:     2026-10-22T04:17:44Z
  Ended:       2026-10-22T04:30:46Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 14:
  State:       Resolved
  Started:     2026-10-22T04:17:45Z
  Ended:       2026-10-22T04:30:46Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 15:
  State:       Resolved
  Started:     2026-10-22T04:17:45Z
  Ended:       2026-10-22T04:30:46Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 16:
  State:       Resolved
  Started:     2026-10-22T04:17:45Z
  Ended:       2026-10-22T04:24:47Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjjhu9ej7evo873ckvf6h944dkcatga", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjjhu9ej7evo873ckvf6h944dkcatga/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 17:
  State:       Resolved
  Started:     2026-10-22T04:17:45Z
  Ended:       2026-10-22T04:28:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjjhr18kgmu4rfp2are2sggc1ihgvv6/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 18:
  State:       Resolved
  Started:     2026-10-22T04:18:47Z
  Ended:       2026-10-22T04:30:46Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 19:
  State:       Resolved
  Started:     2026-10-22T04:18:47Z
  Ended:       2026-10-22T04:26:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 20:
  State:       Resolved
  Started:     2026-10-22T04:18:47Z
  Ended:       2026-10-22T04:29:53Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 21:
  State:       Resolved
  Started:     2026-10-22T04:18:47Z
  Ended:       2026-10-22T04:25:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjjhu9ej7evo873ckvf6h944dkcatga", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjjhu9ej7evo873ckvf6h944dkcatga/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 22:
  State:       Resolved
  Started:     2026-10-22T04:18:47Z
  Ended:       2026-10-22T04:25:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 23:
  State:       Resolved
  Started:     2026-10-22T04:18:47Z
  Ended:       2026-10-22T04:24:47Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 24:
  State:       Resolved
  Started:     2026-10-22T04:18:47Z
  Ended:       2026-10-22T04:30:46Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 25:
  State:       Resolved
  Started:     2026-10-22T04:18:47Z
  Ended:       2026-10-22T04:25:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.
Firing 26:
  State:       Resolved
  Started:     2026-10-22T04:18:47Z
  Ended:       2026-10-22T04:24:47Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 27:
  State:       Resolved
  Started:     2026-10-22T04:18:47Z
  Ended:       2026-10-22T04:30:46Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 28:
  State:       Resolved
  Started:     2026-10-22T04:18:47Z
  Ended:       2026-10-22T04:26:45Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="governance-policy-framework", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjjidblnimv20c5a8vt1qckn86fl4l3/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.
Firing 29:
  State:       Resolved
  Started:     2026-10-22T04:18:47Z
  Ended:       2026-10-22T04:29:53Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="klusterlet-addon-workmgr", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-1", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.
Firing 30:
  State:       Resolved
  Started:     2026-10-22T04:18:47Z
  Ended:       2026-10-22T04:30:46Z
  Severity:    Sev3
  Labels:      alertname="KubeDeploymentReplicasMismatch", cluster="ci01-j6940672-mgmt-1", component="kubernetes-infrastructure", container="kube-state-metrics", deployment="config-policy-controller", endpoint="http", environment="ci01", instance="10.128.64.165:8080", job="kube-state-metrics", microsoft.amwresourceid="/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6940672/providers/microsoft.monitor/accounts/services-j6940672", namespace="klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9", pod="arohcp-monitor-kube-state-metrics-85d48f46ff-vs5d6", prometheus="prometheus/prometheus", prometheus_replica="prom-agent-prometheus-0", region="centralus", service="arohcp-monitor-kube-state-metrics", severity="warning"
  Description: Deployment klusterlet-2tjji5uejciulspvpvv7uusqd81as7e9/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-22T03:40:46Z
  Ended:       2026-10-22T03:51:47Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j6940672-svc", component="backend", name="clusterresources", region="centralus", severity="warning"
  Description: Backend controller workqueue clusterresources has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.

alert fired 1 time(s)
Firing 1:
  State:       Resolved
  Started:     2026-10-22T02:57:07Z
  Ended:       2026-10-22T03:04:04Z
  Severity:    Sev3
  Labels:      alertname="BackendControllerQueueDepthHigh", cluster="ci01-j7474048-svc", component="backend", name="csstatedump", region="westus3", severity="warning"
  Description: Backend controller workqueue csstatedump has had a depth > 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.

fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-23-5h8rfd" with version 4.23.0-ec.0 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x1d0abaa07220>: 
    failed to create HCP cluster cluster-zstream-4-23-5h8rfd: failed waiting for cluster="cluster-zstream-4-23-5h8rfd" in resourcegroup="rg-zstream-upgrade-4-23-5tgvrgshsr5n" to finish creating: GET https://rp.j8224256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: ResourceNotFound
    --------------------------------------------------------------------------------
    {
      "error": {
        "code": "ResourceNotFound",
        "message": "The resource 'locations/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85' under resource group '' was not found.",
        "target": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8224256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85
            correlation ID: f39bfd1a-7a5d-4cc1-a489-fe05576c19eb
            client request ID: 9a94d97d-0e82-4889-a643-43d835a7d044
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster "cluster-zstream-4-23-5h8rfd" with version 4.23.0-ec.0 on candidate channel
Unexpected error:
    <*fmt.wrapError | 0x1d0abaa07220>: 
    failed to create HCP cluster cluster-zstream-4-23-5h8rfd: failed waiting for cluster="cluster-zstream-4-23-5h8rfd" in resourcegroup="rg-zstream-upgrade-4-23-5tgvrgshsr5n" to finish creating: GET https://rp.j8224256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85
    --------------------------------------------------------------------------------
    RESPONSE 404: 404 Not Found
    ERROR CODE: ResourceNotFound
    --------------------------------------------------------------------------------
    {
      "error": {
        "code": "ResourceNotFound",
        "message": "The resource 'locations/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85' under resource group '' was not found.",
        "target": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j8224256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dcf54547-6f2b-4281-953b-a052437c3b85
            correlation ID: f39bfd1a-7a5d-4cc1-a489-fe05576c19eb
            client request ID: 9a94d97d-0e82-4889-a643-43d835a7d044
occurred

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:163]: failed to create node pool
Expected success, but got an error:
    <*fmt.wrapError | 0x1460b7c07300>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "ingress-cert-le" in resourcegroup="ingress-cert-le-twhbflrbjdzn" to finish creating: GET https://rp.j5891200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b8bbf095-2cf3-4399-b879-67ffc150a1c6
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b8bbf095-2cf3-4399-b879-67ffc150a1c6",
      "name": "b8bbf095-2cf3-4399-b879-67ffc150a1c6",
      "status": "Failed",
      "startTime": "2026-10-22T02:18:02.50355295Z",
      "endTime": "2026-10-22T02:37:03.792952128Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine ingress-cert-le-np-1-vg5d4-n6w85: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-twhbflrbjdzn--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-vg5d4-n6w85 to exist\nMachine ingress-cert-le-np-1-vg5d4-wjxjx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-twhbflrbjdzn--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-vg5d4-wjxjx to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine ingress-cert-le-np-1-vg5d4-wjxjx: Failed: availabilitysets failed to create or update. err: failed to get existing resource ingress-cert-le-twhbflrbjdzn--managed/2tji3utlagfblbjct3l9sosa37afh321_ingress-cert-le-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-twhbflrbjdzn--managed/providers/Microsoft.Compute/availabilitySets/2tji3utlagfblbjct3l9sosa37afh321_ingress-cert-le-np-1-as\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '1' sec... message truncated\nMachine ingress-cert-le-np-1-vg5d4-n6w85: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-twhbflrbjdzn--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-vg5d4-n6w85 to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5891200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b8bbf095-2cf3-4399-b879-67ffc150a1c6
            correlation ID: 93044a4f-4151-4014-a170-56a210eccfed
            client request ID: c5d784ea-0b44-447d-be61-01ac35e7274c
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:163]: failed to create node pool
Expected success, but got an error:
    <*fmt.wrapError | 0x1460b7c07300>: 
    failed to create NodePool np-1: failed waiting for nodepool="np-1" for cluster "ingress-cert-le" in resourcegroup="ingress-cert-le-twhbflrbjdzn" to finish creating: GET https://rp.j5891200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b8bbf095-2cf3-4399-b879-67ffc150a1c6
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: DeadlineExceeded
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b8bbf095-2cf3-4399-b879-67ffc150a1c6",
      "name": "b8bbf095-2cf3-4399-b879-67ffc150a1c6",
      "status": "Failed",
      "startTime": "2026-10-22T02:18:02.50355295Z",
      "endTime": "2026-10-22T02:37:03.792952128Z",
      "error": {
        "code": "DeadlineExceeded",
        "message": "node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\nMachine ingress-cert-le-np-1-vg5d4-n6w85: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-twhbflrbjdzn--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-vg5d4-n6w85 to exist\nMachine ingress-cert-le-np-1-vg5d4-wjxjx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-twhbflrbjdzn--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-vg5d4-wjxjx to exist\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\nMachine ingress-cert-le-np-1-vg5d4-wjxjx: Failed: availabilitysets failed to create or update. err: failed to get existing resource ingress-cert-le-twhbflrbjdzn--managed/2tji3utlagfblbjct3l9sosa37afh321_ingress-cert-le-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-twhbflrbjdzn--managed/providers/Microsoft.Compute/availabilitySets/2tji3utlagfblbjct3l9sosa37afh321_ingress-cert-le-np-1-as\n--------------------------------------------------------------------------------\nRESPONSE 429: 429 Too Many Requests\nERROR CODE: SubscriptionRequestsThrottled\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"SubscriptionRequestsThrottled\",\n    \"message\": \"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '1' sec... message truncated\nMachine ingress-cert-le-np-1-vg5d4-n6w85: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-twhbflrbjdzn--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-vg5d4-n6w85 to exist\n"
      }
    }
    --------------------------------------------------------------------------------
    
    <Azure error internals omitted>
        Azure request IDs:
          GET https://rp.j5891200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b8bbf095-2cf3-4399-b879-67ffc150a1c6
            correlation ID: 93044a4f-4151-4014-a170-56a210eccfed
            client request ID: c5d784ea-0b44-447d-be61-01ac35e7274c

fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-47vm96krz6xl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x305427581240>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-47vm96krz6xl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/canadacentral/ci01-j8157440-mgmt-1/ocm-arohcpci01-2tjhsc31bhjko8tujjb78bnnd66r2s41-x9w4k8i6e2b5t4p: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-47vm96krz6xl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/canadacentral/ci01-j8157440-mgmt-1/ocm-arohcpci01-2tjhsc31bhjko8tujjb78bnnd66r2s41-x9w4k8i6e2b5t4p: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        err: <*errors.errorString | 0x305428017c50>{
            s: "dev/canadacentral/ci01-j8157440-mgmt-1/ocm-arohcpci01-2tjhsc31bhjko8tujjb78bnnd66r2s41-x9w4k8i6e2b5t4p: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-47vm96krz6xl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster
Unexpected error:
    <*fmt.wrapError | 0x305427581240>: 
    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-47vm96krz6xl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/canadacentral/ci01-j8157440-mgmt-1/ocm-arohcpci01-2tjhsc31bhjko8tujjb78bnnd66r2s41-x9w4k8i6e2b5t4p: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1
    {
        msg: "VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-47vm96krz6xl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/canadacentral/ci01-j8157440-mgmt-1/ocm-arohcpci01-2tjhsc31bhjko8tujjb78bnnd66r2s41-x9w4k8i6e2b5t4p: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        err: <*errors.errorString | 0x305428017c50>{
            s: "dev/canadacentral/ci01-j8157440-mgmt-1/ocm-arohcpci01-2tjhsc31bhjko8tujjb78bnnd66r2s41-x9w4k8i6e2b5t4p: expected all current routers >= desired replicas > 0; desired=3 current=0 rows=1",
        },
    }
occurred

time=2026-10-22T01:22:39.748Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events description="Step kube-events\n Kind: Helm\n"
time=2026-10-22T01:22:40.100Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events values="map[enabled:true image:map[digest:sha256:f45f2e02544ba4005d39e435073a6c59ab5db372223dee379b08a36d3bc20871 registry:kubernetesshared.azurecr.io repository:shared/kube-events] resources:map[limits:map[memory:unlimited] requests:map[cpu:200m memory:512Mi]]]"
time=2026-10-22T01:22:40.101Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events
time=2026-10-22T01:22:40.245Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events namespace=monitoring
time=2026-10-22T01:22:40.391Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events dryRun=false
time=2026-10-22T01:22:40.391Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events
time=2026-10-22T01:22:40.452Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events
time=2026-10-22T01:27:43.190Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events err="resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\ncontext deadline exceeded"
time=2026-10-22T01:27:43.190Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events err="error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\ncontext deadline exceeded"